blob: e58d9373809dfa3a428f4608fe5e81c2b29687db (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
#ifndef __LINUX_BRLOCK_H
#define __LINUX_BRLOCK_H
/*
* 'Big Reader' read-write spinlocks.
*
* super-fast read/write locks, with write-side penalty. The point
* is to have a per-CPU read/write lock. Readers lock their CPU-local
* readlock, writers must lock all locks to get write access. These
* CPU-read-write locks are semantically identical to normal rwlocks.
* Memory usage is higher as well. (NR_CPUS*L1_CACHE_BYTES bytes)
*
* The most important feature is that these spinlocks do not cause
* cacheline ping-pong in the 'most readonly data' case.
*
* Copyright 2000, Ingo Molnar <mingo@redhat.com>
*
* Registry idea and naming [ crutial! :-) ] by:
*
* David S. Miller <davem@redhat.com>
*
* David has an implementation that doesn't use atomic operations in
* the read branch via memory ordering tricks - i guess we need to
* split this up into a per-arch thing? The atomicity issue is a
* secondary item in profiles, at least on x86 platforms.
*
* The atomic op version overhead is indeed a big deal on
* load-locked/store-conditional cpus (ALPHA/MIPS/PPC) and
* compare-and-swap cpus (Sparc64). So we control which
* implementation to use with a __BRLOCK_USE_ATOMICS define. -DaveM
*/
/* Register bigreader lock indices here. */
enum brlock_indices {
BR_GLOBALIRQ_LOCK,
BR_NETPROTO_LOCK,
__BR_END
};
#include <linux/config.h>
#ifdef CONFIG_SMP
#include <linux/cache.h>
#include <linux/spinlock.h>
#if defined(__i386__) || defined(__ia64__) || defined(__x86_64__)
#define __BRLOCK_USE_ATOMICS
#else
#undef __BRLOCK_USE_ATOMICS
#endif
#ifdef __BRLOCK_USE_ATOMICS
typedef rwlock_t brlock_read_lock_t;
#else
typedef unsigned int brlock_read_lock_t;
#endif
/*
* align last allocated index to the next cacheline:
*/
#define __BR_IDX_MAX \
(((sizeof(brlock_read_lock_t)*__BR_END + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1)) / sizeof(brlock_read_lock_t))
extern brlock_read_lock_t __brlock_array[NR_CPUS][__BR_IDX_MAX];
#ifndef __BRLOCK_USE_ATOMICS
struct br_wrlock {
spinlock_t lock;
} __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
extern struct br_wrlock __br_write_locks[__BR_IDX_MAX];
#endif
extern void __br_lock_usage_bug (void);
#ifdef __BRLOCK_USE_ATOMICS
static inline void br_read_lock (enum brlock_indices idx)
{
/*
* This causes a link-time bug message if an
* invalid index is used:
*/
if (idx >= __BR_END)
__br_lock_usage_bug();
read_lock(&__brlock_array[smp_processor_id()][idx]);
}
static inline void br_read_unlock (enum brlock_indices idx)
{
if (idx >= __BR_END)
__br_lock_usage_bug();
read_unlock(&__brlock_array[smp_processor_id()][idx]);
}
#else /* ! __BRLOCK_USE_ATOMICS */
static inline void br_read_lock (enum brlock_indices idx)
{
unsigned int *ctr;
spinlock_t *lock;
/*
* This causes a link-time bug message if an
* invalid index is used:
*/
if (idx >= __BR_END)
__br_lock_usage_bug();
ctr = &__brlock_array[smp_processor_id()][idx];
lock = &__br_write_locks[idx].lock;
again:
(*ctr)++;
mb();
if (spin_is_locked(lock)) {
(*ctr)--;
wmb(); /*
* The release of the ctr must become visible
* to the other cpus eventually thus wmb(),
* we don't care if spin_is_locked is reordered
* before the releasing of the ctr.
* However IMHO this wmb() is superflous even in theory.
* It would not be superflous only if on the
* other CPUs doing a ldl_l instead of an ldl
* would make a difference and I don't think this is
* the case.
* I'd like to clarify this issue further
* but for now this is a slow path so adding the
* wmb() will keep us on the safe side.
*/
while (spin_is_locked(lock))
barrier();
goto again;
}
}
static inline void br_read_unlock (enum brlock_indices idx)
{
unsigned int *ctr;
if (idx >= __BR_END)
__br_lock_usage_bug();
ctr = &__brlock_array[smp_processor_id()][idx];
wmb();
(*ctr)--;
}
#endif /* __BRLOCK_USE_ATOMICS */
/* write path not inlined - it's rare and larger */
extern void FASTCALL(__br_write_lock (enum brlock_indices idx));
extern void FASTCALL(__br_write_unlock (enum brlock_indices idx));
static inline void br_write_lock (enum brlock_indices idx)
{
if (idx >= __BR_END)
__br_lock_usage_bug();
__br_write_lock(idx);
}
static inline void br_write_unlock (enum brlock_indices idx)
{
if (idx >= __BR_END)
__br_lock_usage_bug();
__br_write_unlock(idx);
}
#else
# define br_read_lock(idx) ((void)(idx))
# define br_read_unlock(idx) ((void)(idx))
# define br_write_lock(idx) ((void)(idx))
# define br_write_unlock(idx) ((void)(idx))
#endif
/*
* Now enumerate all of the possible sw/hw IRQ protected
* versions of the interfaces.
*/
#define br_read_lock_irqsave(idx, flags) \
do { local_irq_save(flags); br_read_lock(idx); } while (0)
#define br_read_lock_irq(idx) \
do { local_irq_disable(); br_read_lock(idx); } while (0)
#define br_read_lock_bh(idx) \
do { local_bh_disable(); br_read_lock(idx); } while (0)
#define br_write_lock_irqsave(idx, flags) \
do { local_irq_save(flags); br_write_lock(idx); } while (0)
#define br_write_lock_irq(idx) \
do { local_irq_disable(); br_write_lock(idx); } while (0)
#define br_write_lock_bh(idx) \
do { local_bh_disable(); br_write_lock(idx); } while (0)
#define br_read_unlock_irqrestore(idx, flags) \
do { br_read_unlock(irx); local_irq_restore(flags); } while (0)
#define br_read_unlock_irq(idx) \
do { br_read_unlock(idx); local_irq_enable(); } while (0)
#define br_read_unlock_bh(idx) \
do { br_read_unlock(idx); local_bh_enable(); } while (0)
#define br_write_unlock_irqrestore(idx, flags) \
do { br_write_unlock(irx); local_irq_restore(flags); } while (0)
#define br_write_unlock_irq(idx) \
do { br_write_unlock(idx); local_irq_enable(); } while (0)
#define br_write_unlock_bh(idx) \
do { br_write_unlock(idx); local_bh_enable(); } while (0)
#endif /* __LINUX_BRLOCK_H */
|
