forked from luck/tmp_suning_uos_patched
eb9b6666d6
The idea of this patch is from Davidlohr Bueso, he posts a patch for bcache to optimize barrier usage for read-modify-write atomic bitops. Indeed such optimization can also apply on other locations where smp_mb() is used before or after an atomic operation. This patch replaces smp_mb() with smp_mb__before_atomic() or smp_mb__after_atomic() in btree.c and writeback.c, where it is used to synchronize memory cache just earlier on other cores. Although the locations are not on hot code path, it is always not bad to mkae things a little better. Signed-off-by: Coly Li <colyli@suse.de> Cc: Davidlohr Bueso <dave@stgolabs.net> Signed-off-by: Jens Axboe <axboe@kernel.dk>
1009 lines
26 KiB
C
1009 lines
26 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* background writeback - scan btree for dirty data and write it to the backing
|
|
* device
|
|
*
|
|
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
|
|
* Copyright 2012 Google, Inc.
|
|
*/
|
|
|
|
#include "bcache.h"
|
|
#include "btree.h"
|
|
#include "debug.h"
|
|
#include "writeback.h"
|
|
|
|
#include <linux/delay.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/sched/clock.h>
|
|
#include <trace/events/bcache.h>
|
|
|
|
static void update_gc_after_writeback(struct cache_set *c)
|
|
{
|
|
if (c->gc_after_writeback != (BCH_ENABLE_AUTO_GC) ||
|
|
c->gc_stats.in_use < BCH_AUTO_GC_DIRTY_THRESHOLD)
|
|
return;
|
|
|
|
c->gc_after_writeback |= BCH_DO_AUTO_GC;
|
|
}
|
|
|
|
/* Rate limiting */
|
|
static uint64_t __calc_target_rate(struct cached_dev *dc)
|
|
{
|
|
struct cache_set *c = dc->disk.c;
|
|
|
|
/*
|
|
* This is the size of the cache, minus the amount used for
|
|
* flash-only devices
|
|
*/
|
|
uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size -
|
|
atomic_long_read(&c->flash_dev_dirty_sectors);
|
|
|
|
/*
|
|
* Unfortunately there is no control of global dirty data. If the
|
|
* user states that they want 10% dirty data in the cache, and has,
|
|
* e.g., 5 backing volumes of equal size, we try and ensure each
|
|
* backing volume uses about 2% of the cache for dirty data.
|
|
*/
|
|
uint32_t bdev_share =
|
|
div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT,
|
|
c->cached_dev_sectors);
|
|
|
|
uint64_t cache_dirty_target =
|
|
div_u64(cache_sectors * dc->writeback_percent, 100);
|
|
|
|
/* Ensure each backing dev gets at least one dirty share */
|
|
if (bdev_share < 1)
|
|
bdev_share = 1;
|
|
|
|
return (cache_dirty_target * bdev_share) >> WRITEBACK_SHARE_SHIFT;
|
|
}
|
|
|
|
static void __update_writeback_rate(struct cached_dev *dc)
|
|
{
|
|
/*
|
|
* PI controller:
|
|
* Figures out the amount that should be written per second.
|
|
*
|
|
* First, the error (number of sectors that are dirty beyond our
|
|
* target) is calculated. The error is accumulated (numerically
|
|
* integrated).
|
|
*
|
|
* Then, the proportional value and integral value are scaled
|
|
* based on configured values. These are stored as inverses to
|
|
* avoid fixed point math and to make configuration easy-- e.g.
|
|
* the default value of 40 for writeback_rate_p_term_inverse
|
|
* attempts to write at a rate that would retire all the dirty
|
|
* blocks in 40 seconds.
|
|
*
|
|
* The writeback_rate_i_inverse value of 10000 means that 1/10000th
|
|
* of the error is accumulated in the integral term per second.
|
|
* This acts as a slow, long-term average that is not subject to
|
|
* variations in usage like the p term.
|
|
*/
|
|
int64_t target = __calc_target_rate(dc);
|
|
int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
|
|
int64_t error = dirty - target;
|
|
int64_t proportional_scaled =
|
|
div_s64(error, dc->writeback_rate_p_term_inverse);
|
|
int64_t integral_scaled;
|
|
uint32_t new_rate;
|
|
|
|
if ((error < 0 && dc->writeback_rate_integral > 0) ||
|
|
(error > 0 && time_before64(local_clock(),
|
|
dc->writeback_rate.next + NSEC_PER_MSEC))) {
|
|
/*
|
|
* Only decrease the integral term if it's more than
|
|
* zero. Only increase the integral term if the device
|
|
* is keeping up. (Don't wind up the integral
|
|
* ineffectively in either case).
|
|
*
|
|
* It's necessary to scale this by
|
|
* writeback_rate_update_seconds to keep the integral
|
|
* term dimensioned properly.
|
|
*/
|
|
dc->writeback_rate_integral += error *
|
|
dc->writeback_rate_update_seconds;
|
|
}
|
|
|
|
integral_scaled = div_s64(dc->writeback_rate_integral,
|
|
dc->writeback_rate_i_term_inverse);
|
|
|
|
new_rate = clamp_t(int32_t, (proportional_scaled + integral_scaled),
|
|
dc->writeback_rate_minimum, NSEC_PER_SEC);
|
|
|
|
dc->writeback_rate_proportional = proportional_scaled;
|
|
dc->writeback_rate_integral_scaled = integral_scaled;
|
|
dc->writeback_rate_change = new_rate -
|
|
atomic_long_read(&dc->writeback_rate.rate);
|
|
atomic_long_set(&dc->writeback_rate.rate, new_rate);
|
|
dc->writeback_rate_target = target;
|
|
}
|
|
|
|
static bool set_at_max_writeback_rate(struct cache_set *c,
|
|
struct cached_dev *dc)
|
|
{
|
|
/* Don't sst max writeback rate if it is disabled */
|
|
if (!c->idle_max_writeback_rate_enabled)
|
|
return false;
|
|
|
|
/* Don't set max writeback rate if gc is running */
|
|
if (!c->gc_mark_valid)
|
|
return false;
|
|
/*
|
|
* Idle_counter is increased everytime when update_writeback_rate() is
|
|
* called. If all backing devices attached to the same cache set have
|
|
* identical dc->writeback_rate_update_seconds values, it is about 6
|
|
* rounds of update_writeback_rate() on each backing device before
|
|
* c->at_max_writeback_rate is set to 1, and then max wrteback rate set
|
|
* to each dc->writeback_rate.rate.
|
|
* In order to avoid extra locking cost for counting exact dirty cached
|
|
* devices number, c->attached_dev_nr is used to calculate the idle
|
|
* throushold. It might be bigger if not all cached device are in write-
|
|
* back mode, but it still works well with limited extra rounds of
|
|
* update_writeback_rate().
|
|
*/
|
|
if (atomic_inc_return(&c->idle_counter) <
|
|
atomic_read(&c->attached_dev_nr) * 6)
|
|
return false;
|
|
|
|
if (atomic_read(&c->at_max_writeback_rate) != 1)
|
|
atomic_set(&c->at_max_writeback_rate, 1);
|
|
|
|
atomic_long_set(&dc->writeback_rate.rate, INT_MAX);
|
|
|
|
/* keep writeback_rate_target as existing value */
|
|
dc->writeback_rate_proportional = 0;
|
|
dc->writeback_rate_integral_scaled = 0;
|
|
dc->writeback_rate_change = 0;
|
|
|
|
/*
|
|
* Check c->idle_counter and c->at_max_writeback_rate agagain in case
|
|
* new I/O arrives during before set_at_max_writeback_rate() returns.
|
|
* Then the writeback rate is set to 1, and its new value should be
|
|
* decided via __update_writeback_rate().
|
|
*/
|
|
if ((atomic_read(&c->idle_counter) <
|
|
atomic_read(&c->attached_dev_nr) * 6) ||
|
|
!atomic_read(&c->at_max_writeback_rate))
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
static void update_writeback_rate(struct work_struct *work)
|
|
{
|
|
struct cached_dev *dc = container_of(to_delayed_work(work),
|
|
struct cached_dev,
|
|
writeback_rate_update);
|
|
struct cache_set *c = dc->disk.c;
|
|
|
|
/*
|
|
* should check BCACHE_DEV_RATE_DW_RUNNING before calling
|
|
* cancel_delayed_work_sync().
|
|
*/
|
|
set_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
|
|
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
|
|
smp_mb__after_atomic();
|
|
|
|
/*
|
|
* CACHE_SET_IO_DISABLE might be set via sysfs interface,
|
|
* check it here too.
|
|
*/
|
|
if (!test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) ||
|
|
test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
|
|
clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
|
|
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
|
|
smp_mb__after_atomic();
|
|
return;
|
|
}
|
|
|
|
if (atomic_read(&dc->has_dirty) && dc->writeback_percent) {
|
|
/*
|
|
* If the whole cache set is idle, set_at_max_writeback_rate()
|
|
* will set writeback rate to a max number. Then it is
|
|
* unncessary to update writeback rate for an idle cache set
|
|
* in maximum writeback rate number(s).
|
|
*/
|
|
if (!set_at_max_writeback_rate(c, dc)) {
|
|
down_read(&dc->writeback_lock);
|
|
__update_writeback_rate(dc);
|
|
update_gc_after_writeback(c);
|
|
up_read(&dc->writeback_lock);
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
* CACHE_SET_IO_DISABLE might be set via sysfs interface,
|
|
* check it here too.
|
|
*/
|
|
if (test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) &&
|
|
!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
|
|
schedule_delayed_work(&dc->writeback_rate_update,
|
|
dc->writeback_rate_update_seconds * HZ);
|
|
}
|
|
|
|
/*
|
|
* should check BCACHE_DEV_RATE_DW_RUNNING before calling
|
|
* cancel_delayed_work_sync().
|
|
*/
|
|
clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
|
|
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
|
|
smp_mb__after_atomic();
|
|
}
|
|
|
|
static unsigned int writeback_delay(struct cached_dev *dc,
|
|
unsigned int sectors)
|
|
{
|
|
if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
|
|
!dc->writeback_percent)
|
|
return 0;
|
|
|
|
return bch_next_delay(&dc->writeback_rate, sectors);
|
|
}
|
|
|
|
struct dirty_io {
|
|
struct closure cl;
|
|
struct cached_dev *dc;
|
|
uint16_t sequence;
|
|
struct bio bio;
|
|
};
|
|
|
|
static void dirty_init(struct keybuf_key *w)
|
|
{
|
|
struct dirty_io *io = w->private;
|
|
struct bio *bio = &io->bio;
|
|
|
|
bio_init(bio, bio->bi_inline_vecs,
|
|
DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS));
|
|
if (!io->dc->writeback_percent)
|
|
bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
|
|
|
|
bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9;
|
|
bio->bi_private = w;
|
|
bch_bio_map(bio, NULL);
|
|
}
|
|
|
|
static void dirty_io_destructor(struct closure *cl)
|
|
{
|
|
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
|
|
|
|
kfree(io);
|
|
}
|
|
|
|
static void write_dirty_finish(struct closure *cl)
|
|
{
|
|
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
|
|
struct keybuf_key *w = io->bio.bi_private;
|
|
struct cached_dev *dc = io->dc;
|
|
|
|
bio_free_pages(&io->bio);
|
|
|
|
/* This is kind of a dumb way of signalling errors. */
|
|
if (KEY_DIRTY(&w->key)) {
|
|
int ret;
|
|
unsigned int i;
|
|
struct keylist keys;
|
|
|
|
bch_keylist_init(&keys);
|
|
|
|
bkey_copy(keys.top, &w->key);
|
|
SET_KEY_DIRTY(keys.top, false);
|
|
bch_keylist_push(&keys);
|
|
|
|
for (i = 0; i < KEY_PTRS(&w->key); i++)
|
|
atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);
|
|
|
|
ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
|
|
|
|
if (ret)
|
|
trace_bcache_writeback_collision(&w->key);
|
|
|
|
atomic_long_inc(ret
|
|
? &dc->disk.c->writeback_keys_failed
|
|
: &dc->disk.c->writeback_keys_done);
|
|
}
|
|
|
|
bch_keybuf_del(&dc->writeback_keys, w);
|
|
up(&dc->in_flight);
|
|
|
|
closure_return_with_destructor(cl, dirty_io_destructor);
|
|
}
|
|
|
|
static void dirty_endio(struct bio *bio)
|
|
{
|
|
struct keybuf_key *w = bio->bi_private;
|
|
struct dirty_io *io = w->private;
|
|
|
|
if (bio->bi_status) {
|
|
SET_KEY_DIRTY(&w->key, false);
|
|
bch_count_backing_io_errors(io->dc, bio);
|
|
}
|
|
|
|
closure_put(&io->cl);
|
|
}
|
|
|
|
static void write_dirty(struct closure *cl)
|
|
{
|
|
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
|
|
struct keybuf_key *w = io->bio.bi_private;
|
|
struct cached_dev *dc = io->dc;
|
|
|
|
uint16_t next_sequence;
|
|
|
|
if (atomic_read(&dc->writeback_sequence_next) != io->sequence) {
|
|
/* Not our turn to write; wait for a write to complete */
|
|
closure_wait(&dc->writeback_ordering_wait, cl);
|
|
|
|
if (atomic_read(&dc->writeback_sequence_next) == io->sequence) {
|
|
/*
|
|
* Edge case-- it happened in indeterminate order
|
|
* relative to when we were added to wait list..
|
|
*/
|
|
closure_wake_up(&dc->writeback_ordering_wait);
|
|
}
|
|
|
|
continue_at(cl, write_dirty, io->dc->writeback_write_wq);
|
|
return;
|
|
}
|
|
|
|
next_sequence = io->sequence + 1;
|
|
|
|
/*
|
|
* IO errors are signalled using the dirty bit on the key.
|
|
* If we failed to read, we should not attempt to write to the
|
|
* backing device. Instead, immediately go to write_dirty_finish
|
|
* to clean up.
|
|
*/
|
|
if (KEY_DIRTY(&w->key)) {
|
|
dirty_init(w);
|
|
bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
|
|
io->bio.bi_iter.bi_sector = KEY_START(&w->key);
|
|
bio_set_dev(&io->bio, io->dc->bdev);
|
|
io->bio.bi_end_io = dirty_endio;
|
|
|
|
/* I/O request sent to backing device */
|
|
closure_bio_submit(io->dc->disk.c, &io->bio, cl);
|
|
}
|
|
|
|
atomic_set(&dc->writeback_sequence_next, next_sequence);
|
|
closure_wake_up(&dc->writeback_ordering_wait);
|
|
|
|
continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
|
|
}
|
|
|
|
static void read_dirty_endio(struct bio *bio)
|
|
{
|
|
struct keybuf_key *w = bio->bi_private;
|
|
struct dirty_io *io = w->private;
|
|
|
|
/* is_read = 1 */
|
|
bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
|
|
bio->bi_status, 1,
|
|
"reading dirty data from cache");
|
|
|
|
dirty_endio(bio);
|
|
}
|
|
|
|
static void read_dirty_submit(struct closure *cl)
|
|
{
|
|
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
|
|
|
|
closure_bio_submit(io->dc->disk.c, &io->bio, cl);
|
|
|
|
continue_at(cl, write_dirty, io->dc->writeback_write_wq);
|
|
}
|
|
|
|
static void read_dirty(struct cached_dev *dc)
|
|
{
|
|
unsigned int delay = 0;
|
|
struct keybuf_key *next, *keys[MAX_WRITEBACKS_IN_PASS], *w;
|
|
size_t size;
|
|
int nk, i;
|
|
struct dirty_io *io;
|
|
struct closure cl;
|
|
uint16_t sequence = 0;
|
|
|
|
BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list));
|
|
atomic_set(&dc->writeback_sequence_next, sequence);
|
|
closure_init_stack(&cl);
|
|
|
|
/*
|
|
* XXX: if we error, background writeback just spins. Should use some
|
|
* mempools.
|
|
*/
|
|
|
|
next = bch_keybuf_next(&dc->writeback_keys);
|
|
|
|
while (!kthread_should_stop() &&
|
|
!test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&
|
|
next) {
|
|
size = 0;
|
|
nk = 0;
|
|
|
|
do {
|
|
BUG_ON(ptr_stale(dc->disk.c, &next->key, 0));
|
|
|
|
/*
|
|
* Don't combine too many operations, even if they
|
|
* are all small.
|
|
*/
|
|
if (nk >= MAX_WRITEBACKS_IN_PASS)
|
|
break;
|
|
|
|
/*
|
|
* If the current operation is very large, don't
|
|
* further combine operations.
|
|
*/
|
|
if (size >= MAX_WRITESIZE_IN_PASS)
|
|
break;
|
|
|
|
/*
|
|
* Operations are only eligible to be combined
|
|
* if they are contiguous.
|
|
*
|
|
* TODO: add a heuristic willing to fire a
|
|
* certain amount of non-contiguous IO per pass,
|
|
* so that we can benefit from backing device
|
|
* command queueing.
|
|
*/
|
|
if ((nk != 0) && bkey_cmp(&keys[nk-1]->key,
|
|
&START_KEY(&next->key)))
|
|
break;
|
|
|
|
size += KEY_SIZE(&next->key);
|
|
keys[nk++] = next;
|
|
} while ((next = bch_keybuf_next(&dc->writeback_keys)));
|
|
|
|
/* Now we have gathered a set of 1..5 keys to write back. */
|
|
for (i = 0; i < nk; i++) {
|
|
w = keys[i];
|
|
|
|
io = kzalloc(sizeof(struct dirty_io) +
|
|
sizeof(struct bio_vec) *
|
|
DIV_ROUND_UP(KEY_SIZE(&w->key),
|
|
PAGE_SECTORS),
|
|
GFP_KERNEL);
|
|
if (!io)
|
|
goto err;
|
|
|
|
w->private = io;
|
|
io->dc = dc;
|
|
io->sequence = sequence++;
|
|
|
|
dirty_init(w);
|
|
bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
|
|
io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
|
|
bio_set_dev(&io->bio,
|
|
PTR_CACHE(dc->disk.c, &w->key, 0)->bdev);
|
|
io->bio.bi_end_io = read_dirty_endio;
|
|
|
|
if (bch_bio_alloc_pages(&io->bio, GFP_KERNEL))
|
|
goto err_free;
|
|
|
|
trace_bcache_writeback(&w->key);
|
|
|
|
down(&dc->in_flight);
|
|
|
|
/*
|
|
* We've acquired a semaphore for the maximum
|
|
* simultaneous number of writebacks; from here
|
|
* everything happens asynchronously.
|
|
*/
|
|
closure_call(&io->cl, read_dirty_submit, NULL, &cl);
|
|
}
|
|
|
|
delay = writeback_delay(dc, size);
|
|
|
|
while (!kthread_should_stop() &&
|
|
!test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&
|
|
delay) {
|
|
schedule_timeout_interruptible(delay);
|
|
delay = writeback_delay(dc, 0);
|
|
}
|
|
}
|
|
|
|
if (0) {
|
|
err_free:
|
|
kfree(w->private);
|
|
err:
|
|
bch_keybuf_del(&dc->writeback_keys, w);
|
|
}
|
|
|
|
/*
|
|
* Wait for outstanding writeback IOs to finish (and keybuf slots to be
|
|
* freed) before refilling again
|
|
*/
|
|
closure_sync(&cl);
|
|
}
|
|
|
|
/* Scan for dirty data */
|
|
|
|
void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned int inode,
|
|
uint64_t offset, int nr_sectors)
|
|
{
|
|
struct bcache_device *d = c->devices[inode];
|
|
unsigned int stripe_offset, stripe, sectors_dirty;
|
|
|
|
if (!d)
|
|
return;
|
|
|
|
if (UUID_FLASH_ONLY(&c->uuids[inode]))
|
|
atomic_long_add(nr_sectors, &c->flash_dev_dirty_sectors);
|
|
|
|
stripe = offset_to_stripe(d, offset);
|
|
stripe_offset = offset & (d->stripe_size - 1);
|
|
|
|
while (nr_sectors) {
|
|
int s = min_t(unsigned int, abs(nr_sectors),
|
|
d->stripe_size - stripe_offset);
|
|
|
|
if (nr_sectors < 0)
|
|
s = -s;
|
|
|
|
if (stripe >= d->nr_stripes)
|
|
return;
|
|
|
|
sectors_dirty = atomic_add_return(s,
|
|
d->stripe_sectors_dirty + stripe);
|
|
if (sectors_dirty == d->stripe_size)
|
|
set_bit(stripe, d->full_dirty_stripes);
|
|
else
|
|
clear_bit(stripe, d->full_dirty_stripes);
|
|
|
|
nr_sectors -= s;
|
|
stripe_offset = 0;
|
|
stripe++;
|
|
}
|
|
}
|
|
|
|
static bool dirty_pred(struct keybuf *buf, struct bkey *k)
|
|
{
|
|
struct cached_dev *dc = container_of(buf,
|
|
struct cached_dev,
|
|
writeback_keys);
|
|
|
|
BUG_ON(KEY_INODE(k) != dc->disk.id);
|
|
|
|
return KEY_DIRTY(k);
|
|
}
|
|
|
|
static void refill_full_stripes(struct cached_dev *dc)
|
|
{
|
|
struct keybuf *buf = &dc->writeback_keys;
|
|
unsigned int start_stripe, stripe, next_stripe;
|
|
bool wrapped = false;
|
|
|
|
stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
|
|
|
|
if (stripe >= dc->disk.nr_stripes)
|
|
stripe = 0;
|
|
|
|
start_stripe = stripe;
|
|
|
|
while (1) {
|
|
stripe = find_next_bit(dc->disk.full_dirty_stripes,
|
|
dc->disk.nr_stripes, stripe);
|
|
|
|
if (stripe == dc->disk.nr_stripes)
|
|
goto next;
|
|
|
|
next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
|
|
dc->disk.nr_stripes, stripe);
|
|
|
|
buf->last_scanned = KEY(dc->disk.id,
|
|
stripe * dc->disk.stripe_size, 0);
|
|
|
|
bch_refill_keybuf(dc->disk.c, buf,
|
|
&KEY(dc->disk.id,
|
|
next_stripe * dc->disk.stripe_size, 0),
|
|
dirty_pred);
|
|
|
|
if (array_freelist_empty(&buf->freelist))
|
|
return;
|
|
|
|
stripe = next_stripe;
|
|
next:
|
|
if (wrapped && stripe > start_stripe)
|
|
return;
|
|
|
|
if (stripe == dc->disk.nr_stripes) {
|
|
stripe = 0;
|
|
wrapped = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Returns true if we scanned the entire disk
|
|
*/
|
|
static bool refill_dirty(struct cached_dev *dc)
|
|
{
|
|
struct keybuf *buf = &dc->writeback_keys;
|
|
struct bkey start = KEY(dc->disk.id, 0, 0);
|
|
struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
|
|
struct bkey start_pos;
|
|
|
|
/*
|
|
* make sure keybuf pos is inside the range for this disk - at bringup
|
|
* we might not be attached yet so this disk's inode nr isn't
|
|
* initialized then
|
|
*/
|
|
if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
|
|
bkey_cmp(&buf->last_scanned, &end) > 0)
|
|
buf->last_scanned = start;
|
|
|
|
if (dc->partial_stripes_expensive) {
|
|
refill_full_stripes(dc);
|
|
if (array_freelist_empty(&buf->freelist))
|
|
return false;
|
|
}
|
|
|
|
start_pos = buf->last_scanned;
|
|
bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
|
|
|
|
if (bkey_cmp(&buf->last_scanned, &end) < 0)
|
|
return false;
|
|
|
|
/*
|
|
* If we get to the end start scanning again from the beginning, and
|
|
* only scan up to where we initially started scanning from:
|
|
*/
|
|
buf->last_scanned = start;
|
|
bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);
|
|
|
|
return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
|
|
}
|
|
|
|
static int bch_writeback_thread(void *arg)
|
|
{
|
|
struct cached_dev *dc = arg;
|
|
struct cache_set *c = dc->disk.c;
|
|
bool searched_full_index;
|
|
|
|
bch_ratelimit_reset(&dc->writeback_rate);
|
|
|
|
while (!kthread_should_stop() &&
|
|
!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
|
|
down_write(&dc->writeback_lock);
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
/*
|
|
* If the bache device is detaching, skip here and continue
|
|
* to perform writeback. Otherwise, if no dirty data on cache,
|
|
* or there is dirty data on cache but writeback is disabled,
|
|
* the writeback thread should sleep here and wait for others
|
|
* to wake up it.
|
|
*/
|
|
if (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
|
|
(!atomic_read(&dc->has_dirty) || !dc->writeback_running)) {
|
|
up_write(&dc->writeback_lock);
|
|
|
|
if (kthread_should_stop() ||
|
|
test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
|
|
set_current_state(TASK_RUNNING);
|
|
break;
|
|
}
|
|
|
|
schedule();
|
|
continue;
|
|
}
|
|
set_current_state(TASK_RUNNING);
|
|
|
|
searched_full_index = refill_dirty(dc);
|
|
|
|
if (searched_full_index &&
|
|
RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
|
|
atomic_set(&dc->has_dirty, 0);
|
|
SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
|
|
bch_write_bdev_super(dc, NULL);
|
|
/*
|
|
* If bcache device is detaching via sysfs interface,
|
|
* writeback thread should stop after there is no dirty
|
|
* data on cache. BCACHE_DEV_DETACHING flag is set in
|
|
* bch_cached_dev_detach().
|
|
*/
|
|
if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {
|
|
up_write(&dc->writeback_lock);
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* When dirty data rate is high (e.g. 50%+), there might
|
|
* be heavy buckets fragmentation after writeback
|
|
* finished, which hurts following write performance.
|
|
* If users really care about write performance they
|
|
* may set BCH_ENABLE_AUTO_GC via sysfs, then when
|
|
* BCH_DO_AUTO_GC is set, garbage collection thread
|
|
* will be wake up here. After moving gc, the shrunk
|
|
* btree and discarded free buckets SSD space may be
|
|
* helpful for following write requests.
|
|
*/
|
|
if (c->gc_after_writeback ==
|
|
(BCH_ENABLE_AUTO_GC|BCH_DO_AUTO_GC)) {
|
|
c->gc_after_writeback &= ~BCH_DO_AUTO_GC;
|
|
force_wake_up_gc(c);
|
|
}
|
|
}
|
|
|
|
up_write(&dc->writeback_lock);
|
|
|
|
read_dirty(dc);
|
|
|
|
if (searched_full_index) {
|
|
unsigned int delay = dc->writeback_delay * HZ;
|
|
|
|
while (delay &&
|
|
!kthread_should_stop() &&
|
|
!test_bit(CACHE_SET_IO_DISABLE, &c->flags) &&
|
|
!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
|
|
delay = schedule_timeout_interruptible(delay);
|
|
|
|
bch_ratelimit_reset(&dc->writeback_rate);
|
|
}
|
|
}
|
|
|
|
if (dc->writeback_write_wq) {
|
|
flush_workqueue(dc->writeback_write_wq);
|
|
destroy_workqueue(dc->writeback_write_wq);
|
|
}
|
|
cached_dev_put(dc);
|
|
wait_for_kthread_stop();
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Init */
|
|
#define INIT_KEYS_EACH_TIME 500000
|
|
#define INIT_KEYS_SLEEP_MS 100
|
|
|
|
struct sectors_dirty_init {
|
|
struct btree_op op;
|
|
unsigned int inode;
|
|
size_t count;
|
|
struct bkey start;
|
|
};
|
|
|
|
static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
|
|
struct bkey *k)
|
|
{
|
|
struct sectors_dirty_init *op = container_of(_op,
|
|
struct sectors_dirty_init, op);
|
|
if (KEY_INODE(k) > op->inode)
|
|
return MAP_DONE;
|
|
|
|
if (KEY_DIRTY(k))
|
|
bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
|
|
KEY_START(k), KEY_SIZE(k));
|
|
|
|
op->count++;
|
|
if (atomic_read(&b->c->search_inflight) &&
|
|
!(op->count % INIT_KEYS_EACH_TIME)) {
|
|
bkey_copy_key(&op->start, k);
|
|
return -EAGAIN;
|
|
}
|
|
|
|
return MAP_CONTINUE;
|
|
}
|
|
|
|
static int bch_root_node_dirty_init(struct cache_set *c,
|
|
struct bcache_device *d,
|
|
struct bkey *k)
|
|
{
|
|
struct sectors_dirty_init op;
|
|
int ret;
|
|
|
|
bch_btree_op_init(&op.op, -1);
|
|
op.inode = d->id;
|
|
op.count = 0;
|
|
op.start = KEY(op.inode, 0, 0);
|
|
|
|
do {
|
|
ret = bcache_btree(map_keys_recurse,
|
|
k,
|
|
c->root,
|
|
&op.op,
|
|
&op.start,
|
|
sectors_dirty_init_fn,
|
|
0);
|
|
if (ret == -EAGAIN)
|
|
schedule_timeout_interruptible(
|
|
msecs_to_jiffies(INIT_KEYS_SLEEP_MS));
|
|
else if (ret < 0) {
|
|
pr_warn("sectors dirty init failed, ret=%d!", ret);
|
|
break;
|
|
}
|
|
} while (ret == -EAGAIN);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int bch_dirty_init_thread(void *arg)
|
|
{
|
|
struct dirty_init_thrd_info *info = arg;
|
|
struct bch_dirty_init_state *state = info->state;
|
|
struct cache_set *c = state->c;
|
|
struct btree_iter iter;
|
|
struct bkey *k, *p;
|
|
int cur_idx, prev_idx, skip_nr;
|
|
int i;
|
|
|
|
k = p = NULL;
|
|
i = 0;
|
|
cur_idx = prev_idx = 0;
|
|
|
|
bch_btree_iter_init(&c->root->keys, &iter, NULL);
|
|
k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad);
|
|
BUG_ON(!k);
|
|
|
|
p = k;
|
|
|
|
while (k) {
|
|
spin_lock(&state->idx_lock);
|
|
cur_idx = state->key_idx;
|
|
state->key_idx++;
|
|
spin_unlock(&state->idx_lock);
|
|
|
|
skip_nr = cur_idx - prev_idx;
|
|
|
|
while (skip_nr) {
|
|
k = bch_btree_iter_next_filter(&iter,
|
|
&c->root->keys,
|
|
bch_ptr_bad);
|
|
if (k)
|
|
p = k;
|
|
else {
|
|
atomic_set(&state->enough, 1);
|
|
/* Update state->enough earlier */
|
|
smp_mb__after_atomic();
|
|
goto out;
|
|
}
|
|
skip_nr--;
|
|
cond_resched();
|
|
}
|
|
|
|
if (p) {
|
|
if (bch_root_node_dirty_init(c, state->d, p) < 0)
|
|
goto out;
|
|
}
|
|
|
|
p = NULL;
|
|
prev_idx = cur_idx;
|
|
cond_resched();
|
|
}
|
|
|
|
out:
|
|
/* In order to wake up state->wait in time */
|
|
smp_mb__before_atomic();
|
|
if (atomic_dec_and_test(&state->started))
|
|
wake_up(&state->wait);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int bch_btre_dirty_init_thread_nr(void)
|
|
{
|
|
int n = num_online_cpus()/2;
|
|
|
|
if (n == 0)
|
|
n = 1;
|
|
else if (n > BCH_DIRTY_INIT_THRD_MAX)
|
|
n = BCH_DIRTY_INIT_THRD_MAX;
|
|
|
|
return n;
|
|
}
|
|
|
|
void bch_sectors_dirty_init(struct bcache_device *d)
|
|
{
|
|
int i;
|
|
struct bkey *k = NULL;
|
|
struct btree_iter iter;
|
|
struct sectors_dirty_init op;
|
|
struct cache_set *c = d->c;
|
|
struct bch_dirty_init_state *state;
|
|
char name[32];
|
|
|
|
/* Just count root keys if no leaf node */
|
|
if (c->root->level == 0) {
|
|
bch_btree_op_init(&op.op, -1);
|
|
op.inode = d->id;
|
|
op.count = 0;
|
|
op.start = KEY(op.inode, 0, 0);
|
|
|
|
for_each_key_filter(&c->root->keys,
|
|
k, &iter, bch_ptr_invalid)
|
|
sectors_dirty_init_fn(&op.op, c->root, k);
|
|
return;
|
|
}
|
|
|
|
state = kzalloc(sizeof(struct bch_dirty_init_state), GFP_KERNEL);
|
|
if (!state) {
|
|
pr_warn("sectors dirty init failed: cannot allocate memory");
|
|
return;
|
|
}
|
|
|
|
state->c = c;
|
|
state->d = d;
|
|
state->total_threads = bch_btre_dirty_init_thread_nr();
|
|
state->key_idx = 0;
|
|
spin_lock_init(&state->idx_lock);
|
|
atomic_set(&state->started, 0);
|
|
atomic_set(&state->enough, 0);
|
|
init_waitqueue_head(&state->wait);
|
|
|
|
for (i = 0; i < state->total_threads; i++) {
|
|
/* Fetch latest state->enough earlier */
|
|
smp_mb__before_atomic();
|
|
if (atomic_read(&state->enough))
|
|
break;
|
|
|
|
state->infos[i].state = state;
|
|
atomic_inc(&state->started);
|
|
snprintf(name, sizeof(name), "bch_dirty_init[%d]", i);
|
|
|
|
state->infos[i].thread =
|
|
kthread_run(bch_dirty_init_thread,
|
|
&state->infos[i],
|
|
name);
|
|
if (IS_ERR(state->infos[i].thread)) {
|
|
pr_err("fails to run thread bch_dirty_init[%d]", i);
|
|
for (--i; i >= 0; i--)
|
|
kthread_stop(state->infos[i].thread);
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
wait_event_interruptible(state->wait,
|
|
atomic_read(&state->started) == 0 ||
|
|
test_bit(CACHE_SET_IO_DISABLE, &c->flags));
|
|
|
|
out:
|
|
kfree(state);
|
|
}
|
|
|
|
void bch_cached_dev_writeback_init(struct cached_dev *dc)
|
|
{
|
|
sema_init(&dc->in_flight, 64);
|
|
init_rwsem(&dc->writeback_lock);
|
|
bch_keybuf_init(&dc->writeback_keys);
|
|
|
|
dc->writeback_metadata = true;
|
|
dc->writeback_running = false;
|
|
dc->writeback_percent = 10;
|
|
dc->writeback_delay = 30;
|
|
atomic_long_set(&dc->writeback_rate.rate, 1024);
|
|
dc->writeback_rate_minimum = 8;
|
|
|
|
dc->writeback_rate_update_seconds = WRITEBACK_RATE_UPDATE_SECS_DEFAULT;
|
|
dc->writeback_rate_p_term_inverse = 40;
|
|
dc->writeback_rate_i_term_inverse = 10000;
|
|
|
|
WARN_ON(test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));
|
|
INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
|
|
}
|
|
|
|
int bch_cached_dev_writeback_start(struct cached_dev *dc)
|
|
{
|
|
dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq",
|
|
WQ_MEM_RECLAIM, 0);
|
|
if (!dc->writeback_write_wq)
|
|
return -ENOMEM;
|
|
|
|
cached_dev_get(dc);
|
|
dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
|
|
"bcache_writeback");
|
|
if (IS_ERR(dc->writeback_thread)) {
|
|
cached_dev_put(dc);
|
|
destroy_workqueue(dc->writeback_write_wq);
|
|
return PTR_ERR(dc->writeback_thread);
|
|
}
|
|
dc->writeback_running = true;
|
|
|
|
WARN_ON(test_and_set_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));
|
|
schedule_delayed_work(&dc->writeback_rate_update,
|
|
dc->writeback_rate_update_seconds * HZ);
|
|
|
|
bch_writeback_queue(dc);
|
|
|
|
return 0;
|
|
}
|