// SPDX-License-Identifier: GPL-2.0-only
/* Network filesystem read subrequest result collection, assessment and
* retrying.
*
* Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
* Written by David Howells ([email protected])
*/
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/task_io_accounting_ops.h>
#include "internal.h"
/*
* Clear the unread part of an I/O request.
*/
static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
{
netfs_reset_iter(subreq);
WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));
iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);
if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
__set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
}
/*
* Flush, mark and unlock a folio that's now completely read. If we want to
* cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
* dirty and let writeback handle it.
*/
static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq,
struct netfs_io_request *rreq,
struct folio_queue *folioq,
int slot)
{
struct netfs_folio *finfo;
struct folio *folio = folioq_folio(folioq, slot);
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
finfo = netfs_folio_info(folio);
if (finfo) {
trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
if (finfo->netfs_group)
folio_change_private(folio, finfo->netfs_group);
else
folio_detach_private(folio);
kfree(finfo);
}
if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) {
if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
folio_mark_dirty(folio);
}
} else {
trace_netfs_folio(folio, netfs_folio_trace_read_done);
}
} else {
// TODO: Use of PG_private_2 is deprecated.
if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))
netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot);
}
if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
if (folio->index == rreq->no_unlock_folio &&
test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
_debug("no unlock");
} else {
trace_netfs_folio(folio, netfs_folio_trace_read_unlock);
folio_unlock(folio);
}
}
}
/*
* Unlock any folios that are now completely read. Returns true if the
* subrequest is removed from the list.
*/
static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async)
{
struct netfs_io_subrequest *prev, *next;
struct netfs_io_request *rreq = subreq->rreq;
struct folio_queue *folioq = subreq->curr_folioq;
size_t avail, prev_donated, next_donated, fsize, part, excess;
loff_t fpos, start;
loff_t fend;
int slot = subreq->curr_folioq_slot;
if (WARN(subreq->transferred > subreq->len,
"Subreq overread: R%x[%x] %zu > %zu",
rreq->debug_id, subreq->debug_index,
subreq->transferred, subreq->len))
subreq->transferred = subreq->len;
next_folio:
fsize = PAGE_SIZE << subreq->curr_folio_order;
fpos = round_down(subreq->start + subreq->consumed, fsize);
fend = fpos + fsize;
if (WARN_ON_ONCE(!folioq) ||
WARN_ON_ONCE(!folioq_folio(folioq, slot)) ||
WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) {
pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n",
rreq->debug_id, subreq->debug_index,
subreq->start, subreq->start + subreq->transferred - 1,
subreq->consumed, subreq->transferred, subreq->len,
slot);
if (folioq) {
struct folio *folio = folioq_folio(folioq, slot);
pr_err("folioq: orders=%02x%02x%02x%02x\n",
folioq->orders[0], folioq->orders[1],
folioq->orders[2], folioq->orders[3]);
if (folio)
pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n",
fpos, fend - 1, folio_pos(folio), folio_order(folio),
folioq_folio_order(folioq, slot));
}
}
donation_changed:
/* Try to consume the current folio if we've hit or passed the end of
* it. There's a possibility that this subreq doesn't start at the
* beginning of the folio, in which case we need to donate to/from the
* preceding subreq.
*
* We also need to include any potential donation back from the
* following subreq.
*/
prev_donated = READ_ONCE(subreq->prev_donated);
next_donated = READ_ONCE(subreq->next_donated);
if (prev_donated || next_donated) {
spin_lock_bh(&rreq->lock);
prev_donated = subreq->prev_donated;
next_donated = subreq->next_donated;
subreq->start -= prev_donated;
subreq->len += prev_donated;
subreq->transferred += prev_donated;
prev_donated = subreq->prev_donated = 0;
if (subreq->transferred == subreq->len) {
subreq->len += next_donated;
subreq->transferred += next_donated;
next_donated = subreq->next_donated = 0;
}
trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations);
spin_unlock_bh(&rreq->lock);
}
avail = subreq->transferred;
if (avail == subreq->len)
avail += next_donated;
start = subreq->start;
if (subreq->consumed == 0) {
start -= prev_donated;
avail += prev_donated;
} else {
start += subreq->consumed;
avail -= subreq->consumed;
}
part = umin(avail, fsize);
trace_netfs_progress(subreq, start, avail, part);
if (start + avail >= fend) {
if (fpos == start) {
/* Flush, unlock and mark for caching any folio we've just read. */
subreq->consumed = fend - subreq->start;
netfs_unlock_read_folio(subreq, rreq, folioq, slot);
folioq_mark2(folioq, slot);
if (subreq->consumed >= subreq->len)
goto remove_subreq;
} else if (fpos < start) {
excess = fend - subreq->start;
spin_lock_bh(&rreq->lock);
/* If we complete first on a folio split with the
* preceding subreq, donate to that subreq - otherwise
* we get the responsibility.
*/
if (subreq->prev_donated != prev_donated) {
spin_unlock_bh(&rreq->lock);
goto donation_changed;
}
if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
spin_unlock_bh(&rreq->lock);
pr_err("Can't donate prior to front\n");
goto bad;
}
prev = list_prev_entry(subreq, rreq_link);
WRITE_ONCE(prev->next_donated, prev->next_donated + excess);
subreq->start += excess;
subreq->len -= excess;
subreq->transferred -= excess;
trace_netfs_donate(rreq, subreq, prev, excess,
netfs_trace_donate_tail_to_prev);
trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
if (subreq->consumed >= subreq->len)
goto remove_subreq_locked;
spin_unlock_bh(&rreq->lock);
} else {
pr_err("fpos > start\n");
goto bad;
}
/* Advance the rolling buffer to the next folio. */
slot++;
if (slot >= folioq_nr_slots(folioq)) {
slot = 0;
folioq = folioq->next;
subreq->curr_folioq = folioq;
}
subreq->curr_folioq_slot = slot;
if (folioq && folioq_folio(folioq, slot))
subreq->curr_folio_order = folioq->orders[slot];
if (!was_async)
cond_resched();
goto next_folio;
}
/* Deal with partial progress. */
if (subreq->transferred < subreq->len)
return false;
/* Donate the remaining downloaded data to one of the neighbouring
* subrequests. Note that we may race with them doing the same thing.
*/
spin_lock_bh(&rreq->lock);
if (subreq->prev_donated != prev_donated ||
subreq->next_donated != next_donated) {
spin_unlock_bh(&rreq->lock);
cond_resched();
goto donation_changed;
}
/* Deal with the trickiest case: that this subreq is in the middle of a
* folio, not touching either edge, but finishes first. In such a
* case, we donate to the previous subreq, if there is one, so that the
* donation is only handled when that completes - and remove this
* subreq from the list.
*
* If the previous subreq finished first, we will have acquired their
* donation and should be able to unlock folios and/or donate nextwards.
*/
if (!subreq->consumed &&
!prev_donated &&
!list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
prev = list_prev_entry(subreq, rreq_link);
WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len);
subreq->start += subreq->len;
subreq->len = 0;
subreq->transferred = 0;
trace_netfs_donate(rreq, subreq, prev, subreq->len,
netfs_trace_donate_to_prev);
trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
goto remove_subreq_locked;
}
/* If we can't donate down the chain, donate up the chain instead. */
excess = subreq->len - subreq->consumed + next_donated;
if (!subreq->consumed)
excess += prev_donated;
if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
rreq->prev_donated = excess;
trace_netfs_donate(rreq, subreq, NULL, excess,
netfs_trace_donate_to_deferred_next);
} else {
next = list_next_entry(subreq, rreq_link);
WRITE_ONCE(next->prev_donated, excess);
trace_netfs_donate(rreq, subreq, next, excess,
netfs_trace_donate_to_next);
}
trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next);
subreq->len = subreq->consumed;
subreq->transferred = subreq->consumed;
goto remove_subreq_locked;
remove_subreq:
spin_lock_bh(&rreq->lock);
remove_subreq_locked:
subreq->consumed = subreq->len;
list_del(&subreq->rreq_link);
spin_unlock_bh(&rreq->lock);
netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed);
return true;
bad:
/* Errr... prev and next both donated to us, but insufficient to finish
* the folio.
*/
printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n",
rreq->debug_id, subreq->debug_index,
subreq->start, subreq->start + subreq->transferred - 1,
subreq->consumed, subreq->transferred, subreq->len);
printk("folio: %llx-%llx\n", fpos, fend - 1);
printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated);
printk("s=%llx av=%zx part=%zx\n", start, avail, part);
BUG();
}
/*
* Do page flushing and suchlike after DIO.
*/
static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
{
struct netfs_io_subrequest *subreq;
unsigned int i;
/* Collect unbuffered reads and direct reads, adding up the transfer
* sizes until we find the first short or failed subrequest.
*/
list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
rreq->transferred += subreq->transferred;
if (subreq->transferred < subreq->len ||
test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {
rreq->error = subreq->error;
break;
}
}
if (rreq->origin == NETFS_DIO_READ) {
for (i = 0; i < rreq->direct_bv_count; i++) {
flush_dcache_page(rreq->direct_bv[i].bv_page);
// TODO: cifs marks pages in the destination buffer
// dirty under some circumstances after a read. Do we
// need to do that too?
set_page_dirty(rreq->direct_bv[i].bv_page);
}
}
if (rreq->iocb) {
rreq->iocb->ki_pos += rreq->transferred;
if (rreq->iocb->ki_complete)
rreq->iocb->ki_complete(
rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
}
if (rreq->netfs_ops->done)
rreq->netfs_ops->done(rreq);
if (rreq->origin == NETFS_DIO_READ)
inode_dio_end(rreq->inode);
}
/*
* Assess the state of a read request and decide what to do next.
*
* Note that we're in normal kernel thread context at this point, possibly
* running on a workqueue.
*/
static void netfs_rreq_assess(struct netfs_io_request *rreq)
{
trace_netfs_rreq(rreq, netfs_rreq_trace_assess);
//netfs_rreq_is_still_valid(rreq);
if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) {
netfs_retry_reads(rreq);
return;
}
if (rreq->origin == NETFS_DIO_READ ||
rreq->origin == NETFS_READ_GAPS)
netfs_rreq_assess_dio(rreq);
task_io_account_read(rreq->transferred);
trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip);
clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
trace_netfs_rreq(rreq, netfs_rreq_trace_done);
netfs_clear_subrequests(rreq, false);
netfs_unlock_abandoned_read_pages(rreq);
if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)))
netfs_pgpriv2_write_to_the_cache(rreq);
}
void netfs_read_termination_worker(struct work_struct *work)
{
struct netfs_io_request *rreq =
container_of(work, struct netfs_io_request, work);
netfs_see_request(rreq, netfs_rreq_trace_see_work);
netfs_rreq_assess(rreq);
netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete);
}
/*
* Handle the completion of all outstanding I/O operations on a read request.
* We inherit a ref from the caller.
*/
void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async)
{
if (!was_async)
return netfs_rreq_assess(rreq);
if (!work_pending(&rreq->work)) {
netfs_get_request(rreq, netfs_rreq_trace_get_work);
if (!queue_work(system_unbound_wq, &rreq->work))
netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq);
}
}
/**
* netfs_read_subreq_progress - Note progress of a read operation.
* @subreq: The read request that has terminated.
* @was_async: True if we're in an asynchronous context.
*
* This tells the read side of netfs lib that a contributory I/O operation has
* made some progress and that it may be possible to unlock some folios.
*
* Before calling, the filesystem should update subreq->transferred to track
* the amount of data copied into the output buffer.
*
* If @was_async is true, the caller might be running in softirq or interrupt
* context and we can't sleep.
*/
void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq,
bool was_async)
{
struct netfs_io_request *rreq = subreq->rreq;
trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
if (subreq->transferred > subreq->consumed &&
(rreq->origin == NETFS_READAHEAD ||
rreq->origin == NETFS_READPAGE ||
rreq->origin == NETFS_READ_FOR_WRITE)) {
netfs_consume_read_data(subreq, was_async);
__clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
}
}
EXPORT_SYMBOL(netfs_read_subreq_progress);
/**
* netfs_read_subreq_terminated - Note the termination of an I/O operation.
* @subreq: The I/O request that has terminated.
* @error: Error code indicating type of completion.
* @was_async: The termination was asynchronous
*
* This tells the read helper that a contributory I/O operation has terminated,
* one way or another, and that it should integrate the results.
*
* The caller indicates the outcome of the operation through @error, supplying
* 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY
* is set) or a negative error code. The helper will look after reissuing I/O
* operations as appropriate and writing downloaded data to the cache.
*
* Before calling, the filesystem should update subreq->transferred to track
* the amount of data copied into the output buffer.
*
* If @was_async is true, the caller might be running in softirq or interrupt
* context and we can't sleep.
*/
void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq,
int error, bool was_async)
{
struct netfs_io_request *rreq = subreq->rreq;
switch (subreq->source) {
case NETFS_READ_FROM_CACHE:
netfs_stat(&netfs_n_rh_read_done);
break;
case NETFS_DOWNLOAD_FROM_SERVER:
netfs_stat(&netfs_n_rh_download_done);
break;
default:
break;
}
if (rreq->origin != NETFS_DIO_READ) {
/* Collect buffered reads.
*
* If the read completed validly short, then we can clear the
* tail before going on to unlock the folios.
*/
if (error == 0 && subreq->transferred < subreq->len &&
(test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) ||
test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) {
netfs_clear_unread(subreq);
subreq->transferred = subreq->len;
trace_netfs_sreq(subreq, netfs_sreq_trace_clear);
}
if (subreq->transferred > subreq->consumed &&
(rreq->origin == NETFS_READAHEAD ||
rreq->origin == NETFS_READPAGE ||
rreq->origin == NETFS_READ_FOR_WRITE)) {
netfs_consume_read_data(subreq, was_async);
__clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
}
rreq->transferred += subreq->transferred;
}
/* Deal with retry requests, short reads and errors. If we retry
* but don't make progress, we abandon the attempt.
*/
if (!error && subreq->transferred < subreq->len) {
if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {
trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof);
} else {
trace_netfs_sreq(subreq, netfs_sreq_trace_short);
if (subreq->transferred > subreq->consumed) {
__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
__clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
} else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) {
__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
} else {
__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
error = -ENODATA;
}
}
}
subreq->error = error;
trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
if (unlikely(error < 0)) {
trace_netfs_failure(rreq, subreq, error, netfs_fail_read);
if (subreq->source == NETFS_READ_FROM_CACHE) {
netfs_stat(&netfs_n_rh_read_failed);
} else {
netfs_stat(&netfs_n_rh_download_failed);
set_bit(NETFS_RREQ_FAILED, &rreq->flags);
rreq->error = subreq->error;
}
}
if (atomic_dec_and_test(&rreq->nr_outstanding))
netfs_rreq_terminated(rreq, was_async);
netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);
}
EXPORT_SYMBOL(netfs_read_subreq_terminated);
Codebase Browser
linux