// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2006 Jens Axboe <[email protected]> * */ #define pr_fmt(fmt) … #include <linux/kernel.h> #include <linux/blkdev.h> #include <linux/blktrace_api.h> #include <linux/percpu.h> #include <linux/init.h> #include <linux/mutex.h> #include <linux/slab.h> #include <linux/debugfs.h> #include <linux/export.h> #include <linux/time.h> #include <linux/uaccess.h> #include <linux/list.h> #include <linux/blk-cgroup.h> #include "../../block/blk.h" #include <trace/events/block.h> #include "trace_output.h" #ifdef CONFIG_BLK_DEV_IO_TRACE static unsigned int blktrace_seq __read_mostly = …; static struct trace_array *blk_tr; static bool blk_tracer_enabled __read_mostly; static LIST_HEAD(running_trace_list); static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock); /* Select an alternative, minimalistic output than the original one */ #define TRACE_BLK_OPT_CLASSIC … #define TRACE_BLK_OPT_CGROUP … #define TRACE_BLK_OPT_CGNAME … static struct tracer_opt blk_tracer_opts[] = …; static struct tracer_flags blk_tracer_flags = …; /* Global reference count of probes */ static DEFINE_MUTEX(blk_probe_mutex); static int blk_probes_ref; static void blk_register_tracepoints(void); static void blk_unregister_tracepoints(void); /* * Send out a notify message. */ static void trace_note(struct blk_trace *bt, pid_t pid, int action, const void *data, size_t len, u64 cgid) { … } /* * Send out a notify for this process, if we haven't done so since a trace * started */ static void trace_note_tsk(struct task_struct *tsk) { … } static void trace_note_time(struct blk_trace *bt) { … } void __blk_trace_note_message(struct blk_trace *bt, struct cgroup_subsys_state *css, const char *fmt, ...) { … } EXPORT_SYMBOL_GPL(…); static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, pid_t pid) { … } /* * Data direction bit lookup */ static const u32 ddir_act[2] = …; #define BLK_TC_RAHEAD … #define BLK_TC_PREFLUSH … /* The ilog2() calls fall out because they're constant */ #define MASK_TC_BIT(rw, __name) … /* * The worker for the various blk_add_trace*() types. Fills out a * blk_io_trace structure and places it in a per-cpu subbuffer. */ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, const blk_opf_t opf, u32 what, int error, int pdu_len, void *pdu_data, u64 cgid) { … } static void blk_trace_free(struct request_queue *q, struct blk_trace *bt) { … } static void get_probe_ref(void) { … } static void put_probe_ref(void) { … } static int blk_trace_start(struct blk_trace *bt) { … } static int blk_trace_stop(struct blk_trace *bt) { … } static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt) { … } static int __blk_trace_remove(struct request_queue *q) { … } int blk_trace_remove(struct request_queue *q) { … } EXPORT_SYMBOL_GPL(…); static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, size_t count, loff_t *ppos) { … } static const struct file_operations blk_dropped_fops = …; static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, size_t count, loff_t *ppos) { … } static const struct file_operations blk_msg_fops = …; /* * Keep track of how many times we encountered a full subbuffer, to aid * the user space app in telling how many lost events there were. */ static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf, void *prev_subbuf, size_t prev_padding) { … } static int blk_remove_buf_file_callback(struct dentry *dentry) { … } static struct dentry *blk_create_buf_file_callback(const char *filename, struct dentry *parent, umode_t mode, struct rchan_buf *buf, int *is_global) { … } static const struct rchan_callbacks blk_relay_callbacks = …; static void blk_trace_setup_lba(struct blk_trace *bt, struct block_device *bdev) { … } /* * Setup everything required to start tracing */ static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, struct block_device *bdev, struct blk_user_trace_setup *buts) { … } static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev, struct block_device *bdev, char __user *arg) { … } int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, struct block_device *bdev, char __user *arg) { … } EXPORT_SYMBOL_GPL(…); #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) static int compat_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, struct block_device *bdev, char __user *arg) { … } #endif static int __blk_trace_startstop(struct request_queue *q, int start) { … } int blk_trace_startstop(struct request_queue *q, int start) { … } EXPORT_SYMBOL_GPL(…); /* * When reading or writing the blktrace sysfs files, the references to the * opened sysfs or device files should prevent the underlying block device * from being removed. So no further delete protection is really needed. */ /** * blk_trace_ioctl - handle the ioctls associated with tracing * @bdev: the block device * @cmd: the ioctl cmd * @arg: the argument data, if any * **/ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) { … } /** * blk_trace_shutdown - stop and cleanup trace structures * @q: the request queue associated with the device * **/ void blk_trace_shutdown(struct request_queue *q) { … } #ifdef CONFIG_BLK_CGROUP static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) { … } #else static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) { return 0; } #endif static u64 blk_trace_request_get_cgid(struct request *rq) { … } /* * blktrace probes */ /** * blk_add_trace_rq - Add a trace for a request oriented action * @rq: the source request * @error: return status to log * @nr_bytes: number of completed bytes * @what: the action * @cgid: the cgroup info * * Description: * Records an action against a request. Will log the bio offset + size. * **/ static void blk_add_trace_rq(struct request *rq, blk_status_t error, unsigned int nr_bytes, u32 what, u64 cgid) { … } static void blk_add_trace_rq_insert(void *ignore, struct request *rq) { … } static void blk_add_trace_rq_issue(void *ignore, struct request *rq) { … } static void blk_add_trace_rq_merge(void *ignore, struct request *rq) { … } static void blk_add_trace_rq_requeue(void *ignore, struct request *rq) { … } static void blk_add_trace_rq_complete(void *ignore, struct request *rq, blk_status_t error, unsigned int nr_bytes) { … } /** * blk_add_trace_bio - Add a trace for a bio oriented action * @q: queue the io is for * @bio: the source bio * @what: the action * @error: error, if any * * Description: * Records an action against a bio. Will log the bio offset + size. * **/ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, u32 what, int error) { … } static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) { … } static void blk_add_trace_bio_complete(void *ignore, struct request_queue *q, struct bio *bio) { … } static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) { … } static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) { … } static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) { … } static void blk_add_trace_getrq(void *ignore, struct bio *bio) { … } static void blk_add_trace_plug(void *ignore, struct request_queue *q) { … } static void blk_add_trace_unplug(void *ignore, struct request_queue *q, unsigned int depth, bool explicit) { … } static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) { … } /** * blk_add_trace_bio_remap - Add a trace for a bio-remap operation * @ignore: trace callback data parameter (not used) * @bio: the source bio * @dev: source device * @from: source sector * * Called after a bio is remapped to a different device and/or sector. **/ static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, sector_t from) { … } /** * blk_add_trace_rq_remap - Add a trace for a request-remap operation * @ignore: trace callback data parameter (not used) * @rq: the source request * @dev: target device * @from: source sector * * Description: * Device mapper remaps request to other devices. * Add a trace for that action. * **/ static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, sector_t from) { … } /** * blk_add_driver_data - Add binary message with driver-specific data * @rq: io request * @data: driver-specific data * @len: length of driver-specific data * * Description: * Some drivers might want to write driver-specific data per request. * **/ void blk_add_driver_data(struct request *rq, void *data, size_t len) { … } EXPORT_SYMBOL_GPL(…); static void blk_register_tracepoints(void) { … } static void blk_unregister_tracepoints(void) { … } /* * struct blk_io_tracer formatting routines */ static void fill_rwbs(char *rwbs, const struct blk_io_trace *t) { … } static inline const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent) { … } static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) { … } static inline u64 t_cgid(const struct trace_entry *ent) { … } static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) { … } static inline u32 t_action(const struct trace_entry *ent) { … } static inline u32 t_bytes(const struct trace_entry *ent) { … } static inline u32 t_sec(const struct trace_entry *ent) { … } static inline unsigned long long t_sector(const struct trace_entry *ent) { … } static inline __u16 t_error(const struct trace_entry *ent) { … } static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) { … } blk_log_action_t; static void blk_log_action_classic(struct trace_iterator *iter, const char *act, bool has_cg) { … } static void blk_log_action(struct trace_iterator *iter, const char *act, bool has_cg) { … } static void blk_log_dump_pdu(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_with_error(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { … } /* * struct tracer operations */ static void blk_tracer_print_header(struct seq_file *m) { … } static void blk_tracer_start(struct trace_array *tr) { … } static int blk_tracer_init(struct trace_array *tr) { … } static void blk_tracer_stop(struct trace_array *tr) { … } static void blk_tracer_reset(struct trace_array *tr) { … } static const struct { … } what2act[] = …; static enum print_line_t print_one_line(struct trace_iterator *iter, bool classic) { … } static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, int flags, struct trace_event *event) { … } static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) { … } static enum print_line_t blk_trace_event_print_binary(struct trace_iterator *iter, int flags, struct trace_event *event) { … } static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) { … } static int blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) { … } static struct tracer blk_tracer __read_mostly = …; static struct trace_event_functions trace_blk_event_funcs = …; static struct trace_event trace_blk_event = …; static int __init init_blk_tracer(void) { … } device_initcall(init_blk_tracer); static int blk_trace_remove_queue(struct request_queue *q) { … } /* * Setup everything required to start tracing */ static int blk_trace_setup_queue(struct request_queue *q, struct block_device *bdev) { … } /* * sysfs interface to enable and configure tracing */ static ssize_t sysfs_blk_trace_attr_show(struct device *dev, struct device_attribute *attr, char *buf); static ssize_t sysfs_blk_trace_attr_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count); #define BLK_TRACE_DEVICE_ATTR(_name) … static BLK_TRACE_DEVICE_ATTR(enable); static BLK_TRACE_DEVICE_ATTR(act_mask); static BLK_TRACE_DEVICE_ATTR(pid); static BLK_TRACE_DEVICE_ATTR(start_lba); static BLK_TRACE_DEVICE_ATTR(end_lba); static struct attribute *blk_trace_attrs[] = …; struct attribute_group blk_trace_attr_group = …; static const struct { … } mask_maps[] = …; static int blk_trace_str2mask(const char *str) { … } static ssize_t blk_trace_mask2str(char *buf, int mask) { … } static ssize_t sysfs_blk_trace_attr_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static ssize_t sysfs_blk_trace_attr_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } #endif /* CONFIG_BLK_DEV_IO_TRACE */ #ifdef CONFIG_EVENT_TRACING /** * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string. * @rwbs: buffer to be filled * @opf: request operation type (REQ_OP_XXX) and flags for the tracepoint * * Description: * Maps each request operation and flag to a single character and fills the * buffer provided by the caller with resulting string. * **/ void blk_fill_rwbs(char *rwbs, blk_opf_t opf) { … } EXPORT_SYMBOL_GPL(…); #endif /* CONFIG_EVENT_TRACING */
Codebase Browser
linux