// SPDX-License-Identifier: GPL-2.0 /* * padata.c - generic interface to process data streams in parallel * * See Documentation/core-api/padata.rst for more information. * * Copyright (C) 2008, 2009 secunet Security Networks AG * Copyright (C) 2008, 2009 Steffen Klassert <[email protected]> * * Copyright (c) 2020 Oracle and/or its affiliates. * Author: Daniel Jordan <[email protected]> */ #include <linux/completion.h> #include <linux/export.h> #include <linux/cpumask.h> #include <linux/err.h> #include <linux/cpu.h> #include <linux/padata.h> #include <linux/mutex.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/sysfs.h> #include <linux/rcupdate.h> #define PADATA_WORK_ONSTACK … struct padata_work { … }; static DEFINE_SPINLOCK(padata_works_lock); static struct padata_work *padata_works; static LIST_HEAD(padata_free_works); struct padata_mt_job_state { … }; static void padata_free_pd(struct parallel_data *pd); static void __init padata_mt_helper(struct work_struct *work); static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) { … } static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr) { … } static struct padata_work *padata_work_alloc(void) { … } /* * This function is marked __ref because this function may be optimized in such * a way that it directly refers to work_fn's address, which causes modpost to * complain when work_fn is marked __init. This scenario was observed with clang * LTO, where padata_work_init() was optimized to refer directly to * padata_mt_helper() because the calls to padata_work_init() with other work_fn * values were eliminated or inlined. */ static void __ref padata_work_init(struct padata_work *pw, work_func_t work_fn, void *data, int flags) { … } static int __init padata_work_alloc_mt(int nworks, void *data, struct list_head *head) { … } static void padata_work_free(struct padata_work *pw) { … } static void __init padata_works_free(struct list_head *works) { … } static void padata_parallel_worker(struct work_struct *parallel_work) { … } /** * padata_do_parallel - padata parallelization function * * @ps: padatashell * @padata: object to be parallelized * @cb_cpu: pointer to the CPU that the serialization callback function should * run on. If it's not in the serial cpumask of @pinst * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if * none found, returns -EINVAL. * * The parallelization callback function will run with BHs off. * Note: Every object which is parallelized by padata_do_parallel * must be seen by padata_do_serial. * * Return: 0 on success or else negative error code. */ int padata_do_parallel(struct padata_shell *ps, struct padata_priv *padata, int *cb_cpu) { … } EXPORT_SYMBOL(…); /* * padata_find_next - Find the next object that needs serialization. * * Return: * * A pointer to the control struct of the next object that needs * serialization, if present in one of the percpu reorder queues. * * NULL, if the next object that needs serialization will * be parallel processed by another cpu and is not yet present in * the cpu's reorder queue. */ static struct padata_priv *padata_find_next(struct parallel_data *pd, bool remove_object) { … } static void padata_reorder(struct parallel_data *pd) { … } static void invoke_padata_reorder(struct work_struct *work) { … } static void padata_serial_worker(struct work_struct *serial_work) { … } /** * padata_do_serial - padata serialization function * * @padata: object to be serialized. * * padata_do_serial must be called for every parallelized object. * The serialization callback function will run with BHs off. */ void padata_do_serial(struct padata_priv *padata) { … } EXPORT_SYMBOL(…); static int padata_setup_cpumasks(struct padata_instance *pinst) { … } static void __init padata_mt_helper(struct work_struct *w) { … } /** * padata_do_multithreaded - run a multithreaded job * @job: Description of the job. * * See the definition of struct padata_mt_job for more details. */ void __init padata_do_multithreaded(struct padata_mt_job *job) { … } static void __padata_list_init(struct padata_list *pd_list) { … } /* Initialize all percpu queues used by serial workers */ static void padata_init_squeues(struct parallel_data *pd) { … } /* Initialize per-CPU reorder lists */ static void padata_init_reorder_list(struct parallel_data *pd) { … } /* Allocate and initialize the internal cpumask dependend resources. */ static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) { … } static void padata_free_pd(struct parallel_data *pd) { … } static void __padata_start(struct padata_instance *pinst) { … } static void __padata_stop(struct padata_instance *pinst) { … } /* Replace the internal control structure with a new one. */ static int padata_replace_one(struct padata_shell *ps) { … } static int padata_replace(struct padata_instance *pinst) { … } /* If cpumask contains no active cpu, we mark the instance as invalid. */ static bool padata_validate_cpumask(struct padata_instance *pinst, const struct cpumask *cpumask) { … } static int __padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, cpumask_var_t cbcpumask) { … } /** * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value * equivalent to @cpumask. * @pinst: padata instance * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding * to parallel and serial cpumasks respectively. * @cpumask: the cpumask to use * * Return: 0 on success or negative error code */ int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask) { … } EXPORT_SYMBOL(…); #ifdef CONFIG_HOTPLUG_CPU static int __padata_add_cpu(struct padata_instance *pinst, int cpu) { … } static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) { … } static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) { … } static int padata_cpu_online(unsigned int cpu, struct hlist_node *node) { … } static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node) { … } static enum cpuhp_state hp_online; #endif static void __padata_free(struct padata_instance *pinst) { … } #define kobj2pinst(_kobj) … #define attr2pentry(_attr) … static void padata_sysfs_release(struct kobject *kobj) { … } struct padata_sysfs_entry { … }; static ssize_t show_cpumask(struct padata_instance *pinst, struct attribute *attr, char *buf) { … } static ssize_t store_cpumask(struct padata_instance *pinst, struct attribute *attr, const char *buf, size_t count) { … } #define PADATA_ATTR_RW(_name, _show_name, _store_name) … #define PADATA_ATTR_RO(_name, _show_name) … PADATA_ATTR_RW(…); PADATA_ATTR_RW(…); /* * Padata sysfs provides the following objects: * serial_cpumask [RW] - cpumask for serial workers * parallel_cpumask [RW] - cpumask for parallel workers */ static struct attribute *padata_default_attrs[] = …; ATTRIBUTE_GROUPS(…); static ssize_t padata_sysfs_show(struct kobject *kobj, struct attribute *attr, char *buf) { … } static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { … } static const struct sysfs_ops padata_sysfs_ops = …; static const struct kobj_type padata_attr_type = …; /** * padata_alloc - allocate and initialize a padata instance * @name: used to identify the instance * * Return: new instance on success, NULL on error */ struct padata_instance *padata_alloc(const char *name) { … } EXPORT_SYMBOL(…); /** * padata_free - free a padata instance * * @pinst: padata instance to free */ void padata_free(struct padata_instance *pinst) { … } EXPORT_SYMBOL(…); /** * padata_alloc_shell - Allocate and initialize padata shell. * * @pinst: Parent padata_instance object. * * Return: new shell on success, NULL on error */ struct padata_shell *padata_alloc_shell(struct padata_instance *pinst) { … } EXPORT_SYMBOL(…); /** * padata_free_shell - free a padata shell * * @ps: padata shell to free */ void padata_free_shell(struct padata_shell *ps) { … } EXPORT_SYMBOL(…); void __init padata_init(void) { … }
Codebase Browser
linux