-
Notifications
You must be signed in to change notification settings - Fork 1
/
pmtlock-v3.0.patch
150 lines (138 loc) · 4.88 KB
/
pmtlock-v3.0.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 64b6117..7288db0 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -62,7 +62,15 @@ static inline void __ticket_unlock_kick(arch_spinlock_t *lock,
#endif /* CONFIG_PARAVIRT_SPINLOCKS */
static inline int __tickets_equal(__ticket_t one, __ticket_t two)
{
- return !((one ^ two) & ~TICKET_SLOWPATH_FLAG);
+ return !((one ^ two) & ~LOCK_HOLD_FLAG);
+}
+
+extern u64 pmtlock_timeout_shift;
+static inline u64 __timeout_threshold(__ticket_t head, __ticket_t tail)
+{
+ // the (__ticket_t) trick handles tail wraparound case
+ // e.g. (unsigned char)(1 - 255) = 2
+ return (__ticket_t)(tail - head) << pmtlock_timeout_shift;
}
static inline void __ticket_check_and_clear_slowpath(arch_spinlock_t *lock,
@@ -102,24 +110,35 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
{
register struct __raw_tickets inc = { .tail = TICKET_LOCK_INC };
+ u64 timeout = 0;
+ __ticket_t current_head;
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail))
- goto out;
+ if (likely(__tickets_equal(inc.head, inc.tail)))
+ goto spin;
+
+
+ timeout = __timeout_threshold(inc.head, inc.tail);
+ do {
+ current_head = ACCESS_ONCE(lock->tickets.head);
+ if (__tickets_equal(current_head, inc.tail)) {
+ goto spin;
+ }
+ cpu_relax();
+ } while (timeout--);
+spin:
for (;;) {
- unsigned count = SPIN_THRESHOLD;
-
- do {
- inc.head = READ_ONCE(lock->tickets.head);
- if (__tickets_equal(inc.head, inc.tail))
- goto clear_slowpath;
- cpu_relax();
- } while (--count);
- __ticket_lock_spinning(lock, inc.tail);
- }
-clear_slowpath:
- __ticket_check_and_clear_slowpath(lock, inc.head);
+ current_head = ACCESS_ONCE(lock->tickets.head);
+ if (likely((current_head & LOCK_HOLD_FLAG) == 0)
+ && cmpxchg(&lock->tickets.head,
+ current_head & ~LOCK_HOLD_FLAG,
+ current_head | LOCK_HOLD_FLAG)
+ == (current_head & ~LOCK_HOLD_FLAG))
+ goto out;
+ cpu_relax();
+ }
+
out:
barrier(); /* make sure nothing creeps before the lock is taken */
}
@@ -133,7 +152,7 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
return 0;
new.head_tail = old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT);
- new.head_tail &= ~TICKET_SLOWPATH_FLAG;
+ new.tickets.head |= LOCK_HOLD_FLAG;
/* cmpxchg is a full barrier, so nothing can move before it */
return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
@@ -141,34 +160,22 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
{
- if (TICKET_SLOWPATH_FLAG &&
- static_key_false(¶virt_ticketlocks_enabled)) {
- __ticket_t head;
-
- BUILD_BUG_ON(((__ticket_t)NR_CPUS) != NR_CPUS);
-
- head = xadd(&lock->tickets.head, TICKET_LOCK_INC);
-
- if (unlikely(head & TICKET_SLOWPATH_FLAG)) {
- head &= ~TICKET_SLOWPATH_FLAG;
- __ticket_unlock_kick(lock, (head + TICKET_LOCK_INC));
- }
- } else
- __add(&lock->tickets.head, TICKET_LOCK_INC, UNLOCK_LOCK_PREFIX);
+ // xadd 1 works because +2 equals set LOCK_HOLD_FLAG and plus 1
+ __add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
}
static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
struct __raw_tickets tmp = READ_ONCE(lock->tickets);
- return !__tickets_equal(tmp.tail, tmp.head);
+ return !__tickets_equal(tmp.tail, tmp.head) || ((tmp.head | LOCK_HOLD_FLAG) != 0);
}
static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
struct __raw_tickets tmp = READ_ONCE(lock->tickets);
- tmp.head &= ~TICKET_SLOWPATH_FLAG;
+ tmp.head &= ~LOCK_HOLD_FLAG;
return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;
}
#define arch_spin_is_contended arch_spin_is_contended
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 5f9d757..7d671aa 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -7,8 +7,9 @@
#define __TICKET_LOCK_INC 2
#define TICKET_SLOWPATH_FLAG ((__ticket_t)1)
#else
-#define __TICKET_LOCK_INC 1
+#define __TICKET_LOCK_INC 2
#define TICKET_SLOWPATH_FLAG ((__ticket_t)0)
+#define LOCK_HOLD_FLAG ((__ticket_t)1)
#endif
#if (CONFIG_NR_CPUS < (256 / __TICKET_LOCK_INC))
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index db3ccb1..4971d3d 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -21,6 +21,11 @@
#include <linux/debug_locks.h>
#include <linux/export.h>
+// timeout threshold for waiter N = N * 2 ^ pmtlock_timeout_shift
+// smaller shift value yields worse fairness and better performance
+u64 pmtlock_timeout_shift = 8;
+EXPORT_SYMBOL(pmtlock_timeout_shift);
+
/*
* If lockdep is enabled then we use the non-preemption spin-ops
* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are