rcu: Add synchronize_srcu_expedited()

This patch creates a synchronize_srcu_expedited() that uses
synchronize_sched_expedited() where synchronize_srcu()
uses synchronize_sched().  The synchronize_srcu() and
synchronize_srcu_expedited() functions become one-liners that
pass synchronize_sched() or synchronize_sched_expedited(),
repectively, to a new __synchronize_srcu() function.

While in the file, move the EXPORT_SYMBOL_GPL()s to immediately
follow the corresponding functions.

Requested-by: Avi Kivity <avi@redhat.com>
Tested-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: Josh Triplett <josh@joshtriplett.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
Cc: avi@redhat.com
LKML-Reference: <12565226354038-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Paul E. McKenney 2009-10-25 19:03:51 -07:00 committed by Ingo Molnar
parent 9b1d82fa16
commit 0cd397d336
2 changed files with 52 additions and 23 deletions

View File

@ -48,6 +48,7 @@ void cleanup_srcu_struct(struct srcu_struct *sp);
int srcu_read_lock(struct srcu_struct *sp) __acquires(sp); int srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
void synchronize_srcu(struct srcu_struct *sp); void synchronize_srcu(struct srcu_struct *sp);
void synchronize_srcu_expedited(struct srcu_struct *sp);
long srcu_batches_completed(struct srcu_struct *sp); long srcu_batches_completed(struct srcu_struct *sp);
#endif #endif

View File

@ -49,6 +49,7 @@ int init_srcu_struct(struct srcu_struct *sp)
sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array); sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array);
return (sp->per_cpu_ref ? 0 : -ENOMEM); return (sp->per_cpu_ref ? 0 : -ENOMEM);
} }
EXPORT_SYMBOL_GPL(init_srcu_struct);
/* /*
* srcu_readers_active_idx -- returns approximate number of readers * srcu_readers_active_idx -- returns approximate number of readers
@ -97,6 +98,7 @@ void cleanup_srcu_struct(struct srcu_struct *sp)
free_percpu(sp->per_cpu_ref); free_percpu(sp->per_cpu_ref);
sp->per_cpu_ref = NULL; sp->per_cpu_ref = NULL;
} }
EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
/** /**
* srcu_read_lock - register a new reader for an SRCU-protected structure. * srcu_read_lock - register a new reader for an SRCU-protected structure.
@ -118,6 +120,7 @@ int srcu_read_lock(struct srcu_struct *sp)
preempt_enable(); preempt_enable();
return idx; return idx;
} }
EXPORT_SYMBOL_GPL(srcu_read_lock);
/** /**
* srcu_read_unlock - unregister a old reader from an SRCU-protected structure. * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
@ -136,22 +139,12 @@ void srcu_read_unlock(struct srcu_struct *sp, int idx)
per_cpu_ptr(sp->per_cpu_ref, smp_processor_id())->c[idx]--; per_cpu_ptr(sp->per_cpu_ref, smp_processor_id())->c[idx]--;
preempt_enable(); preempt_enable();
} }
EXPORT_SYMBOL_GPL(srcu_read_unlock);
/** /*
* synchronize_srcu - wait for prior SRCU read-side critical-section completion * Helper function for synchronize_srcu() and synchronize_srcu_expedited().
* @sp: srcu_struct with which to synchronize.
*
* Flip the completed counter, and wait for the old count to drain to zero.
* As with classic RCU, the updater must use some separate means of
* synchronizing concurrent updates. Can block; must be called from
* process context.
*
* Note that it is illegal to call synchornize_srcu() from the corresponding
* SRCU read-side critical section; doing so will result in deadlock.
* However, it is perfectly legal to call synchronize_srcu() on one
* srcu_struct from some other srcu_struct's read-side critical section.
*/ */
void synchronize_srcu(struct srcu_struct *sp) void __synchronize_srcu(struct srcu_struct *sp, void (*sync_func)(void))
{ {
int idx; int idx;
@ -173,7 +166,7 @@ void synchronize_srcu(struct srcu_struct *sp)
return; return;
} }
synchronize_sched(); /* Force memory barrier on all CPUs. */ sync_func(); /* Force memory barrier on all CPUs. */
/* /*
* The preceding synchronize_sched() ensures that any CPU that * The preceding synchronize_sched() ensures that any CPU that
@ -190,7 +183,7 @@ void synchronize_srcu(struct srcu_struct *sp)
idx = sp->completed & 0x1; idx = sp->completed & 0x1;
sp->completed++; sp->completed++;
synchronize_sched(); /* Force memory barrier on all CPUs. */ sync_func(); /* Force memory barrier on all CPUs. */
/* /*
* At this point, because of the preceding synchronize_sched(), * At this point, because of the preceding synchronize_sched(),
@ -203,7 +196,7 @@ void synchronize_srcu(struct srcu_struct *sp)
while (srcu_readers_active_idx(sp, idx)) while (srcu_readers_active_idx(sp, idx))
schedule_timeout_interruptible(1); schedule_timeout_interruptible(1);
synchronize_sched(); /* Force memory barrier on all CPUs. */ sync_func(); /* Force memory barrier on all CPUs. */
/* /*
* The preceding synchronize_sched() forces all srcu_read_unlock() * The preceding synchronize_sched() forces all srcu_read_unlock()
@ -236,6 +229,47 @@ void synchronize_srcu(struct srcu_struct *sp)
mutex_unlock(&sp->mutex); mutex_unlock(&sp->mutex);
} }
/**
* synchronize_srcu - wait for prior SRCU read-side critical-section completion
* @sp: srcu_struct with which to synchronize.
*
* Flip the completed counter, and wait for the old count to drain to zero.
* As with classic RCU, the updater must use some separate means of
* synchronizing concurrent updates. Can block; must be called from
* process context.
*
* Note that it is illegal to call synchronize_srcu() from the corresponding
* SRCU read-side critical section; doing so will result in deadlock.
* However, it is perfectly legal to call synchronize_srcu() on one
* srcu_struct from some other srcu_struct's read-side critical section.
*/
void synchronize_srcu(struct srcu_struct *sp)
{
__synchronize_srcu(sp, synchronize_sched);
}
EXPORT_SYMBOL_GPL(synchronize_srcu);
/**
* synchronize_srcu_expedited - like synchronize_srcu, but less patient
* @sp: srcu_struct with which to synchronize.
*
* Flip the completed counter, and wait for the old count to drain to zero.
* As with classic RCU, the updater must use some separate means of
* synchronizing concurrent updates. Can block; must be called from
* process context.
*
* Note that it is illegal to call synchronize_srcu_expedited()
* from the corresponding SRCU read-side critical section; doing so
* will result in deadlock. However, it is perfectly legal to call
* synchronize_srcu_expedited() on one srcu_struct from some other
* srcu_struct's read-side critical section.
*/
void synchronize_srcu_expedited(struct srcu_struct *sp)
{
__synchronize_srcu(sp, synchronize_sched_expedited);
}
EXPORT_SYMBOL_GPL(synchronize_srcu_expedited);
/** /**
* srcu_batches_completed - return batches completed. * srcu_batches_completed - return batches completed.
* @sp: srcu_struct on which to report batch completion. * @sp: srcu_struct on which to report batch completion.
@ -248,10 +282,4 @@ long srcu_batches_completed(struct srcu_struct *sp)
{ {
return sp->completed; return sp->completed;
} }
EXPORT_SYMBOL_GPL(init_srcu_struct);
EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
EXPORT_SYMBOL_GPL(srcu_read_lock);
EXPORT_SYMBOL_GPL(srcu_read_unlock);
EXPORT_SYMBOL_GPL(synchronize_srcu);
EXPORT_SYMBOL_GPL(srcu_batches_completed); EXPORT_SYMBOL_GPL(srcu_batches_completed);