/*

* mm/page-writeback.c

*

* Copyright (C) 2002, Linus Torvalds.

* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <[email protected]>

*

* Contains functions related to writing back dirty pages at the

* address_space level.

*

* 10Apr2002 Andrew Morton

* Initial version

*/

#include <linux/kernel.h>

#include <linux/export.h>

#include <linux/spinlock.h>

#include <linux/fs.h>

#include <linux/mm.h>

#include <linux/swap.h>

#include <linux/slab.h>

#include <linux/pagemap.h>

#include <linux/writeback.h>

#include <linux/init.h>

#include <linux/backing-dev.h>

#include <linux/task_io_accounting_ops.h>

#include <linux/blkdev.h>

#include <linux/mpage.h>

#include <linux/rmap.h>

#include <linux/percpu.h>

#include <linux/notifier.h>

#include <linux/smp.h>

#include <linux/sysctl.h>

#include <linux/cpu.h>

#include <linux/syscalls.h>

#include <linux/buffer_head.h> /* __set_page_dirty_buffers */

#include <linux/pagevec.h>

#include <trace/events/writeback.h>

/*

* Sleep at most 200ms at a time in balance_dirty_pages().

*/

#define MAX_PAUSE max(HZ/5, 1)

/*

* Try to keep balance_dirty_pages() call intervals higher than this many pages

* by raising pause time to max_pause when falls below it.

*/

#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))

/*

* Estimate write bandwidth at 200ms intervals.

*/

#define BANDWIDTH_INTERVAL max(HZ/5, 1)

#define RATELIMIT_CALC_SHIFT 10

/*

* After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited

* will look to see if it needs to force writeback or throttling.

*/

static long ratelimit_pages = 32;

/* The following parameters are exported via /proc/sys/vm */

/*

* Start background writeback (via writeback threads) at this percentage

*/

int dirty_background_ratio = 10;

/*

* dirty_background_bytes starts at 0 (disabled) so that it is a function of

* dirty_background_ratio * the amount of dirtyable memory

*/

unsigned long dirty_background_bytes;

/*

* free highmem will not be subtracted from the total free memory

* for calculating free ratios if vm_highmem_is_dirtyable is true

*/

int vm_highmem_is_dirtyable;

/*

* The generator of dirty data starts writeback at this percentage

*/

int vm_dirty_ratio = 20;

/*

* vm_dirty_bytes starts at 0 (disabled) so that it is a function of

* vm_dirty_ratio * the amount of dirtyable memory

*/

unsigned long vm_dirty_bytes;

/*

* The interval between `kupdate'-style writebacks

*/

unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */

EXPORT_SYMBOL_GPL(dirty_writeback_interval);

/*

* The longest time for which data is allowed to remain dirty

*/

unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */

/*

* Flag that makes the machine dump writes/reads and block dirtyings.

*/

int block_dump;

/*

* Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:

* a full sync is triggered after this time elapses without any disk activity.

*/

int laptop_mode;

EXPORT_SYMBOL(laptop_mode);

/* End of sysctl-exported parameters */

unsigned long global_dirty_limit;

/*

* Scale the writeback cache size proportional to the relative writeout speeds.

*

* We do this by keeping a floating proportion between BDIs, based on page

* writeback completions [end_page_writeback()]. Those devices that write out

* pages fastest will get the larger share, while the slower will get a smaller

* share.

*

* We use page writeout completions because we are interested in getting rid of

* dirty pages. Having them written out is the primary goal.

*

* We introduce a concept of time, a period over which we measure these events,

* because demand can/will vary over time. The length of this period itself is

* measured in page writeback completions.

*

*/

static struct prop_descriptor vm_completions;

/*

* Work out the current dirty-memory clamping and background writeout

* thresholds.

*

* The main aim here is to lower them aggressively if there is a lot of mapped

* memory around. To avoid stressing page reclaim with lots of unreclaimable

* pages. It is better to clamp down on writers than to start swapping, and

* performing lots of scanning.

*

* We only allow 1/2 of the currently-unmapped memory to be dirtied.

*

* We don't permit the clamping level to fall below 5% - that is getting rather

* excessive.

*

* We make sure that the background writeout level is below the adjusted

* clamping level.

*/

/*

* In a memory zone, there is a certain amount of pages we consider

* available for the page cache, which is essentially the number of

* free and reclaimable pages, minus some zone reserves to protect

* lowmem and the ability to uphold the zone's watermarks without

* requiring writeback.

*

* This number of dirtyable pages is the base value of which the

* user-configurable dirty ratio is the effictive number of pages that

* are allowed to be actually dirtied. Per individual zone, or

* globally by using the sum of dirtyable pages over all zones.

*

* Because the user is allowed to specify the dirty limit globally as

* absolute number of bytes, calculating the per-zone dirty limit can

* require translating the configured limit into a percentage of

* global dirtyable memory first.

*/

static unsigned long highmem_dirtyable_memory(unsigned long total)

{

#ifdef CONFIG_HIGHMEM

int node;

unsigned long x = 0;

for_each_node_state(node, N_HIGH_MEMORY) {

struct zone *z =

&NODE_DATA(node)->node_zones[ZONE_HIGHMEM];

x += zone_page_state(z, NR_FREE_PAGES) +

zone_reclaimable_pages(z) - z->dirty_balance_reserve;

}

/*

* Make sure that the number of highmem pages is never larger

* than the number of the total dirtyable memory. This can only

* occur in very strange VM situations but we want to make sure

* that this does not occur.

*/

return min(x, total);

#else

return 0;

#endif

}

/**

* global_dirtyable_memory - number of globally dirtyable pages

*

* Returns the global number of pages potentially available for dirty

* page cache. This is the base value for the global dirty limits.

*/

unsigned long global_dirtyable_memory(void)

{

unsigned long x;

x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages() -

dirty_balance_reserve;

if (!vm_highmem_is_dirtyable)

x -= highmem_dirtyable_memory(x);

return x + 1; /* Ensure that we never return 0 */

}

/*

* global_dirty_limits - background-writeback and dirty-throttling thresholds

*

* Calculate the dirty thresholds based on sysctl parameters

* - vm.dirty_background_ratio or vm.dirty_background_bytes

* - vm.dirty_ratio or vm.dirty_bytes

* The dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and

* real-time tasks.

*/

void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)

{

unsigned long background;

unsigned long dirty;

unsigned long uninitialized_var(available_memory);

struct task_struct *tsk;

if (!vm_dirty_bytes || !dirty_background_bytes)

available_memory = global_dirtyable_memory();

if (vm_dirty_bytes)

dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE);

else

dirty = (vm_dirty_ratio * available_memory) / 100;

if (dirty_background_bytes)

background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE);

else

background = (dirty_background_ratio * available_memory) / 100;

if (background >= dirty)

background = dirty / 2;

tsk = current;

if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {

background += background / 4;

dirty += dirty / 4;

}

*pbackground = background;

*pdirty = dirty;

trace_global_dirty_state(background, dirty);

}

/**

* zone_dirtyable_memory - number of dirtyable pages in a zone

* @zone: the zone

*

* Returns the zone's number of pages potentially available for dirty

* page cache. This is the base value for the per-zone dirty limits.

*/

static unsigned long zone_dirtyable_memory(struct zone *zone)

{

/*

* The effective global number of dirtyable pages may exclude

* highmem as a big-picture measure to keep the ratio between

* dirty memory and lowmem reasonable.

*

* But this function is purely about the individual zone and a

* highmem zone can hold its share of dirty pages, so we don't

* care about vm_highmem_is_dirtyable here.

*/

return zone_page_state(zone, NR_FREE_PAGES) +

zone_reclaimable_pages(zone) -

zone->dirty_balance_reserve;

}

/**

* zone_dirty_limit - maximum number of dirty pages allowed in a zone

* @zone: the zone

*

* Returns the maximum number of dirty pages allowed in a zone, based

* on the zone's dirtyable memory.

*/

static unsigned long zone_dirty_limit(struct zone *zone)

{

unsigned long zone_memory = zone_dirtyable_memory(zone);

struct task_struct *tsk = current;

unsigned long dirty;

if (vm_dirty_bytes)

dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *

zone_memory / global_dirtyable_memory();

else

dirty = vm_dirty_ratio * zone_memory / 100;

if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk))

dirty += dirty / 4;

return dirty;

}

/**

* zone_dirty_ok - tells whether a zone is within its dirty limits

* @zone: the zone to check

*

* Returns %true when the dirty pages in @zone are within the zone's

* dirty limit, %false if the limit is exceeded.

*/

bool zone_dirty_ok(struct zone *zone)

{

unsigned long limit = zone_dirty_limit(zone);

return zone_page_state(zone, NR_FILE_DIRTY) +

zone_page_state(zone, NR_UNSTABLE_NFS) +

zone_page_state(zone, NR_WRITEBACK) <= limit;

}

/*

* couple the period to the dirty_ratio:

*

* period/2 ~ roundup_pow_of_two(dirty limit)

*/

static int calc_period_shift(void)

{

unsigned long dirty_total;

if (vm_dirty_bytes)

dirty_total = vm_dirty_bytes / PAGE_SIZE;

else

dirty_total = (vm_dirty_ratio * global_dirtyable_memory()) /

100;

return 2 + ilog2(dirty_total - 1);

}

/*

* update the period when the dirty threshold changes.

*/

static void update_completion_period(void)

{

int shift = calc_period_shift();

prop_change_shift(&vm_completions, shift);

writeback_set_ratelimit();

}

int dirty_background_ratio_handler(struct ctl_table *table, int write,

void __user *buffer, size_t *lenp,

loff_t *ppos)

{

int ret;

ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

if (ret == 0 && write)

dirty_background_bytes = 0;

return ret;

}

int dirty_background_bytes_handler(struct ctl_table *table, int write,

void __user *buffer, size_t *lenp,

loff_t *ppos)

{

int ret;

ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

if (ret == 0 && write)

dirty_background_ratio = 0;

return ret;

}

int dirty_ratio_handler(struct ctl_table *table, int write,

void __user *buffer, size_t *lenp,

loff_t *ppos)

{

int old_ratio = vm_dirty_ratio;

int ret;

ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

if (ret == 0 && write && vm_dirty_ratio != old_ratio) {

update_completion_period();

vm_dirty_bytes = 0;

}

return ret;

}

int dirty_bytes_handler(struct ctl_table *table, int write,

void __user *buffer, size_t *lenp,

loff_t *ppos)

{

unsigned long old_bytes = vm_dirty_bytes;

int ret;

ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

if (ret == 0 && write && vm_dirty_bytes != old_bytes) {

update_completion_period();

vm_dirty_ratio = 0;

}

return ret;

}

/*

* Increment the BDI's writeout completion count and the global writeout

* completion count. Called from test_clear_page_writeback().

*/

static inline void __bdi_writeout_inc(struct backing_dev_info *bdi)

{

__inc_bdi_stat(bdi, BDI_WRITTEN);

__prop_inc_percpu_max(&vm_completions, &bdi->completions,

bdi->max_prop_frac);

}

void bdi_writeout_inc(struct backing_dev_info *bdi)

{

unsigned long flags;

local_irq_save(flags);

__bdi_writeout_inc(bdi);

local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(bdi_writeout_inc);

/*

* Obtain an accurate fraction of the BDI's portion.

*/

static void bdi_writeout_fraction(struct backing_dev_info *bdi,

long *numerator, long *denominator)

{

prop_fraction_percpu(&vm_completions, &bdi->completions,

numerator, denominator);

}

/*

* bdi_min_ratio keeps the sum of the minimum dirty shares of all

* registered backing devices, which, for obvious reasons, can not

* exceed 100%.

*/

static unsigned int bdi_min_ratio;

int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)

{

int ret = 0;

spin_lock_bh(&bdi_lock);

if (min_ratio > bdi->max_ratio) {

ret = -EINVAL;

} else {

min_ratio -= bdi->min_ratio;

if (bdi_min_ratio + min_ratio < 100) {

bdi_min_ratio += min_ratio;

bdi->min_ratio += min_ratio;

} else {

ret = -EINVAL;

}

}

spin_unlock_bh(&bdi_lock);

return ret;

}

int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)

{

int ret = 0;

if (max_ratio > 100)

return -EINVAL;

spin_lock_bh(&bdi_lock);

if (bdi->min_ratio > max_ratio) {

ret = -EINVAL;

} else {

bdi->max_ratio = max_ratio;

bdi->max_prop_frac = (PROP_FRAC_BASE * max_ratio) / 100;

}

spin_unlock_bh(&bdi_lock);

return ret;

}

EXPORT_SYMBOL(bdi_set_max_ratio);

static unsigned long dirty_freerun_ceiling(unsigned long thresh,

unsigned long bg_thresh)

{

return (thresh + bg_thresh) / 2;

}

static unsigned long hard_dirty_limit(unsigned long thresh)

{

return max(thresh, global_dirty_limit);

}

/**

* bdi_dirty_limit - @bdi's share of dirty throttling threshold

* @bdi: the backing_dev_info to query

* @dirty: global dirty limit in pages

*

* Returns @bdi's dirty limit in pages. The term "dirty" in the context of

* dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.

*

* Note that balance_dirty_pages() will only seriously take it as a hard limit

* when sleeping max_pause per page is not enough to keep the dirty pages under

* control. For example, when the device is completely stalled due to some error

* conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.

* In the other normal situations, it acts more gently by throttling the tasks

* more (rather than completely block them) when the bdi dirty pages go high.

*

* It allocates high/low dirty limits to fast/slow devices, in order to prevent

* - starving fast devices

* - piling up dirty pages (that will take long time to sync) on slow devices

*

* The bdi's share of dirty limit will be adapting to its throughput and

* bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.

*/

unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)

{

u64 bdi_dirty;

long numerator, denominator;

/*

* Calculate this BDI's share of the dirty ratio.

*/

bdi_writeout_fraction(bdi, &numerator, &denominator);

bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;

bdi_dirty *= numerator;

do_div(bdi_dirty, denominator);

bdi_dirty += (dirty * bdi->min_ratio) / 100;

if (bdi_dirty > (dirty * bdi->max_ratio) / 100)

bdi_dirty = dirty * bdi->max_ratio / 100;

return bdi_dirty;

}

/*

* Dirty position control.

*

* (o) global/bdi setpoints

*

* We want the dirty pages be balanced around the global/bdi setpoints.

* When the number of dirty pages is higher/lower than the setpoint, the

* dirty position control ratio (and hence task dirty ratelimit) will be

* decreased/increased to bring the dirty pages back to the setpoint.

*

* pos_ratio = 1 << RATELIMIT_CALC_SHIFT

*

* if (dirty < setpoint) scale up pos_ratio

* if (dirty > setpoint) scale down pos_ratio

*

* if (bdi_dirty < bdi_setpoint) scale up pos_ratio

* if (bdi_dirty > bdi_setpoint) scale down pos_ratio

*

* task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT

*

* (o) global control line

*

* ^ pos_ratio

* |

* | |<===== global dirty control scope ======>|

* 2.0 .............*

* | .*

* | . *

* | . *

* | . *

* | . *

* | . *

* 1.0 ................................*

* | . . *

* | . . *

* | . . *

* | . . *

* | . . *

* 0 +------------.------------------.----------------------*------------->

* freerun^ setpoint^ limit^ dirty pages

*

* (o) bdi control line

*

* ^ pos_ratio

* |

* | *

* | *

* | *

* | *

* | * |<=========== span ============>|

* 1.0 .......................*

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* | . *

* 1/4 ...............................................* * * * * * * * * * * *

* | . .

* | . .

* | . .

* 0 +----------------------.-------------------------------.------------->

* bdi_setpoint^ x_intercept^

*

* The bdi control line won't drop below pos_ratio=1/4, so that bdi_dirty can

* be smoothly throttled down to normal if it starts high in situations like

* - start writing to a slow SD card and a fast disk at the same time. The SD

* card's bdi_dirty may rush to many times higher than bdi_setpoint.

* - the bdi dirty thresh drops quickly due to change of JBOD workload

*/

static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,

unsigned long thresh,

unsigned long bg_thresh,

unsigned long dirty,

unsigned long bdi_thresh,

unsigned long bdi_dirty)

{

unsigned long write_bw = bdi->avg_write_bandwidth;

unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);

unsigned long limit = hard_dirty_limit(thresh);

unsigned long x_intercept;

unsigned long setpoint; /* dirty pages' target balance point */

unsigned long bdi_setpoint;

unsigned long span;

long long pos_ratio; /* for scaling up/down the rate limit */

long x;

if (unlikely(dirty >= limit))

return 0;

/*

* global setpoint

*

* setpoint - dirty 3

* f(dirty) := 1.0 + (----------------)

* limit - setpoint

*

* it's a 3rd order polynomial that subjects to

*

* (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast

* (2) f(setpoint) = 1.0 => the balance point

* (3) f(limit) = 0 => the hard limit

* (4) df/dx <= 0 => negative feedback control

* (5) the closer to setpoint, the smaller |df/dx| (and the reverse)

* => fast response on large errors; small oscillation near setpoint

*/

setpoint = (freerun + limit) / 2;

x = div_s64((setpoint - dirty) << RATELIMIT_CALC_SHIFT,

limit - setpoint + 1);

pos_ratio = x;

pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;

pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;

pos_ratio += 1 << RATELIMIT_CALC_SHIFT;

/*

* We have computed basic pos_ratio above based on global situation. If

* the bdi is over/under its share of dirty pages, we want to scale

* pos_ratio further down/up. That is done by the following mechanism.

*/

/*

* bdi setpoint

*

* f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint)

*

* x_intercept - bdi_dirty

* := --------------------------

* x_intercept - bdi_setpoint

*

* The main bdi control line is a linear function that subjects to

*

* (1) f(bdi_setpoint) = 1.0

* (2) k = - 1 / (8 * write_bw) (in single bdi case)

* or equally: x_intercept = bdi_setpoint + 8 * write_bw

*

* For single bdi case, the dirty pages are observed to fluctuate

* regularly within range

* [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2]

* for various filesystems, where (2) can yield in a reasonable 12.5%

* fluctuation range for pos_ratio.

*

* For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its

* own size, so move the slope over accordingly and choose a slope that

* yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh.

*/

if (unlikely(bdi_thresh > thresh))

bdi_thresh = thresh;

/*

* It's very possible that bdi_thresh is close to 0 not because the

* device is slow, but that it has remained inactive for long time.

* Honour such devices a reasonable good (hopefully IO efficient)

* threshold, so that the occasional writes won't be blocked and active

* writes can rampup the threshold quickly.

*/

bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);

/*

* scale global setpoint to bdi's:

* bdi_setpoint = setpoint * bdi_thresh / thresh

*/

x = div_u64((u64)bdi_thresh << 16, thresh + 1);

bdi_setpoint = setpoint * (u64)x >> 16;

/*

* Use span=(8*write_bw) in single bdi case as indicated by

* (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case.

*

* bdi_thresh thresh - bdi_thresh

* span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh

* thresh thresh

*/

span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16;

x_intercept = bdi_setpoint + span;

if (bdi_dirty < x_intercept - span / 4) {

pos_ratio = div_u64(pos_ratio * (x_intercept - bdi_dirty),

x_intercept - bdi_setpoint + 1);

} else

pos_ratio /= 4;

/*

* bdi reserve area, safeguard against dirty pool underrun and disk idle

* It may push the desired control point of global dirty pages higher

* than setpoint.

*/

x_intercept = bdi_thresh / 2;

if (bdi_dirty < x_intercept) {

if (bdi_dirty > x_intercept / 8)

pos_ratio = div_u64(pos_ratio * x_intercept, bdi_dirty);

else

pos_ratio *= 8;

}

return pos_ratio;

}

static void bdi_update_write_bandwidth(struct backing_dev_info *bdi,

unsigned long elapsed,

unsigned long written)

{

const unsigned long period = roundup_pow_of_two(3 * HZ);

unsigned long avg = bdi->avg_write_bandwidth;

unsigned long old = bdi->write_bandwidth;

u64 bw;

/*

* bw = written * HZ / elapsed

*

* bw * elapsed + write_bandwidth * (period - elapsed)

* write_bandwidth = ---------------------------------------------------

* period

*/

bw = written - bdi->written_stamp;

bw *= HZ;

if (unlikely(elapsed > period)) {

do_div(bw, elapsed);

avg = bw;

goto out;

}

bw += (u64)bdi->write_bandwidth * (period - elapsed);

bw >>= ilog2(period);

/*

* one more level of smoothing, for filtering out sudden spikes

*/

if (avg > old && old >= (unsigned long)bw)

avg -= (avg - old) >> 3;

if (avg < old && old <= (unsigned long)bw)

avg += (old - avg) >> 3;

out:

bdi->write_bandwidth = bw;

bdi->avg_write_bandwidth = avg;

}

/*

* The global dirtyable memory and dirty threshold could be suddenly knocked

* down by a large amount (eg. on the startup of KVM in a swapless system).

* This may throw the system into deep dirty exceeded state and throttle

* heavy/light dirtiers alike. To retain good responsiveness, maintain

* global_dirty_limit for tracking slowly down to the knocked down dirty

* threshold.

*/

static void update_dirty_limit(unsigned long thresh, unsigned long dirty)

{

unsigned long limit = global_dirty_limit;

/*

* Follow up in one step.

*/

if (limit < thresh) {

limit = thresh;

goto update;

}

/*

* Follow down slowly. Use the higher one as the target, because thresh

* may drop below dirty. This is exactly the reason to introduce

* global_dirty_limit which is guaranteed to lie above the dirty pages.

*/

thresh = max(thresh, dirty);

if (limit > thresh) {

limit -= (limit - thresh) >> 5;

goto update;

}

return;

update:

global_dirty_limit = limit;

}

static void global_update_bandwidth(unsigned long thresh,

unsigned long dirty,

unsigned long now)

{

static DEFINE_SPINLOCK(dirty_lock);

static unsigned long update_time;

/*

* check locklessly first to optimize away locking for the most time

*/

if (time_before(now, update_time + BANDWIDTH_INTERVAL))

return;

spin_lock(&dirty_lock);

if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) {

update_dirty_limit(thresh, dirty);

update_time = now;

}

spin_unlock(&dirty_lock);

}

/*

* Maintain bdi->dirty_ratelimit, the base dirty throttle rate.

*

* Normal bdi tasks will be curbed at or below it in long term.

* Obviously it should be around (write_bw / N) when there are N dd tasks.

*/

static void bdi_update_dirty_ratelimit(struct backing_dev_info *bdi,

unsigned long thresh,

unsigned long bg_thresh,

unsigned long dirty,

unsigned long bdi_thresh,

unsigned long bdi_dirty,

unsigned long dirtied,

unsigned long elapsed)

{

unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);

unsigned long limit = hard_dirty_limit(thresh);

unsigned long setpoint = (freerun + limit) / 2;

unsigned long write_bw = bdi->avg_write_bandwidth;

unsigned long dirty_ratelimit = bdi->dirty_ratelimit;

unsigned long dirty_rate;

unsigned long task_ratelimit;

unsigned long balanced_dirty_ratelimit;

unsigned long pos_ratio;

unsigned long step;

unsigned long x;

/*

* The dirty rate will match the writeout rate in long term, except

* when dirty pages are truncated by userspace or re-dirtied by FS.

*/

dirty_rate = (dirtied - bdi->dirtied_stamp) * HZ / elapsed;

pos_ratio = bdi_position_ratio(bdi, thresh, bg_thresh, dirty,

bdi_thresh, bdi_dirty);

/*

* task_ratelimit reflects each dd's dirty rate for the past 200ms.

*/

task_ratelimit = (u64)dirty_ratelimit *

pos_ratio >> RATELIMIT_CALC_SHIFT;

task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */

/*

* A linear estimation of the "balanced" throttle rate. The theory is,

* if there are N dd tasks, each throttled at task_ratelimit, the bdi's

* dirty_rate will be measured to be (N * task_ratelimit). So the below

* formula will yield the balanced rate limit (write_bw / N).

*

* Note that the expanded form is not a pure rate feedback:

* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1)

* but also takes pos_ratio into account:

* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2)

*

* (1) is not realistic because pos_ratio also takes part in balancing

* the dirty rate. Consider the state

* pos_ratio = 0.5 (3)

* rate = 2 * (write_bw / N) (4)

* If (1) is used, it will stuck in that state! Because each dd will

* be throttled at

* task_ratelimit = pos_ratio * rate = (write_bw / N) (5)

* yielding

* dirty_rate = N * task_ratelimit = write_bw (6)

* put (6) into (1) we get

* rate_(i+1) = rate_(i) (7)

*

* So we end up using (2) to always keep

* rate_(i+1) ~= (write_bw / N) (8)

* regardless of the value of pos_ratio. As long as (8) is satisfied,

* pos_ratio is able to drive itself to 1.0, which is not only where

* the dirty count meet the setpoint, but also where the slope of

* pos_ratio is most flat and hence task_ratelimit is least fluctuated.

*/

balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,

dirty_rate | 1);

/*

* balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw

*/

if (unlikely(balanced_dirty_ratelimit > write_bw))

balanced_dirty_ratelimit = write_bw;

/*

* We could safely do this and return immediately:

*

* bdi->dirty_ratelimit = balanced_dirty_ratelimit;

*

* However to get a more stable dirty_ratelimit, the below elaborated

* code makes use of task_ratelimit to filter out sigular points and

* limit the step size.

*

* The below code essentially only uses the relative value of

*

* task_ratelimit - dirty_ratelimit

* = (pos_ratio - 1) * dirty_ratelimit

*

* which reflects the direction and size of dirty position error.

*/

/*

* dirty_ratelimit will follow balanced_dirty_ratelimit iff

* task_ratelimit is on the same side of dirty_ratelimit, too.

* For example, when

* - dirty_ratelimit > balanced_dirty_ratelimit

* - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)

* lowering dirty_ratelimit will help meet both the position and rate

* control targets. Otherwise, don't update dirty_ratelimit if it will

* only help meet the rate target. After all, what the users ultimately

* feel and care are stable dirty rate and small position error.

*

* |task_ratelimit - dirty_ratelimit| is used to limit the step size

* and filter out the sigular points of balanced_dirty_ratelimit. Which

* keeps jumping around randomly and can even leap far away at times

* due to the small 200ms estimation period of dirty_rate (we want to

* keep that period small to reduce time lags).

*/

step = 0;

if (dirty < setpoint) {

x = min(bdi->balanced_dirty_ratelimit,

min(balanced_dirty_ratelimit, task_ratelimit));

if (dirty_ratelimit < x)

step = x - dirty_ratelimit;

} else {

x = max(bdi->balanced_dirty_ratelimit,

max(balanced_dirty_ratelimit, task_ratelimit));

if (dirty_ratelimit > x)

step = dirty_ratelimit - x;

}

/*

* Don't pursue 100% rate matching. It's impossible since the balanced

* rate itself is constantly fluctuating. So decrease the track speed

* when it gets close to the target. Helps eliminate pointless tremors.

*/

step >>= dirty_ratelimit / (2 * step + 1);

/*

* Limit the tracking speed to avoid overshooting.

*/

step = (step + 7) / 8;

if (dirty_ratelimit < balanced_dirty_ratelimit)

dirty_ratelimit += step;

else

dirty_ratelimit -= step;

bdi->dirty_ratelimit = max(dirty_ratelimit, 1UL);

bdi->balanced_dirty_ratelimit = balanced_dirty_ratelimit;

trace_bdi_dirty_ratelimit(bdi, dirty_rate, task_ratelimit);

}

void __bdi_update_bandwidth(struct backing_dev_info *bdi,

unsigned long thresh,

unsigned long bg_thresh,

unsigned long dirty,

unsigned long bdi_thresh,

unsigned long bdi_dirty,

unsigned long start_time)

{

unsigned long now = jiffies;

unsigned long elapsed = now - bdi->bw_time_stamp;

unsigned long dirtied;

unsigned long written;

/*

* rate-limit, only update once every 200ms.

*/

if (elapsed < BANDWIDTH_INTERVAL)