From fa28360ea78d1ca19b8799a2076e585040c8e126 Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Mon, 27 Jan 2020 18:21:09 +0100 Subject: [PATCH 01/48] ZEN: INTERACTIVE: Enable background reclaim of hugepages Use [defer+madvise] as default khugepaged defrag strategy: For some reason, the default strategy to respond to THP fault fallbacks is still just madvise, meaning stall if the program wants transparent hugepages, but don't trigger a background reclaim / compaction if THP begins to fail allocations. This creates a snowball affect where we still use the THP code paths, but we almost always fail once a system has been active and busy for a while. The option "defer" was created for interactive systems where THP can still improve performance. If we have to fallback to a regular page due to an allocation failure or anything else, we will trigger a background reclaim and compaction so future THP attempts succeed and previous attempts eventually have their smaller pages combined without stalling running applications. We still want madvise to stall applications that explicitely want THP, so defer+madvise _does_ make a ton of sense. Make it the default for interactive systems, especially if the kernel maintainer left transparent hugepages on "always". Reasoning and details in the original patch: https://lwn.net/Articles/711248/ Signed-off-by: Kai Krakow --- mm/huge_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6301ecc1f679a7..5883c3cbacbd2c 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -53,7 +53,7 @@ unsigned long transparent_hugepage_flags __read_mostly = #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1< Date: Mon, 2 Nov 2020 18:41:38 -0500 Subject: [PATCH 02/48] Revert "futex: Remove needless goto's" This reverts commit d7c5ed73b19c4640426d9c106f70ec2cb532034d. Signed-off-by: Kai Krakow --- kernel/futex.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/kernel/futex.c b/kernel/futex.c index 98a6e1b80bfe46..b978a76b8a5b1d 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1600,13 +1600,13 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ); if (unlikely(ret != 0)) - return ret; + goto out; hb = hash_futex(&key); /* Make sure we really have tasks to wakeup */ if (!hb_waiters_pending(hb)) - return ret; + goto out; spin_lock(&hb->lock); @@ -1629,6 +1629,7 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) spin_unlock(&hb->lock); wake_up_q(&wake_q); +out: return ret; } @@ -1695,10 +1696,10 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, retry: ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ); if (unlikely(ret != 0)) - return ret; + goto out; ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE); if (unlikely(ret != 0)) - return ret; + goto out; hb1 = hash_futex(&key1); hb2 = hash_futex(&key2); @@ -1716,13 +1717,13 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, * an MMU, but we might get them from range checking */ ret = op_ret; - return ret; + goto out; } if (op_ret == -EFAULT) { ret = fault_in_user_writeable(uaddr2); if (ret) - return ret; + goto out; } if (!(flags & FLAGS_SHARED)) { @@ -1765,6 +1766,7 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, out_unlock: double_unlock_hb(hb1, hb2); wake_up_q(&wake_q); +out: return ret; } @@ -1971,18 +1973,20 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, retry: ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ); if (unlikely(ret != 0)) - return ret; + goto out; ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, requeue_pi ? FUTEX_WRITE : FUTEX_READ); if (unlikely(ret != 0)) - return ret; + goto out; /* * The check above which compares uaddrs is not sufficient for * shared futexes. We need to compare the keys: */ - if (requeue_pi && match_futex(&key1, &key2)) - return -EINVAL; + if (requeue_pi && match_futex(&key1, &key2)) { + ret = -EINVAL; + goto out; + } hb1 = hash_futex(&key1); hb2 = hash_futex(&key2); @@ -2002,7 +2006,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, ret = get_user(curval, uaddr1); if (ret) - return ret; + goto out; if (!(flags & FLAGS_SHARED)) goto retry_private; @@ -2068,7 +2072,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, ret = fault_in_user_writeable(uaddr2); if (!ret) goto retry; - return ret; + goto out; case -EBUSY: case -EAGAIN: /* @@ -2187,6 +2191,8 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, double_unlock_hb(hb1, hb2); wake_up_q(&wake_q); hb_waiters_dec(hb2); + +out: return ret ? ret : task_count; } @@ -2665,7 +2671,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, ret = get_user(uval, uaddr); if (ret) - return ret; + goto out; if (!(flags & FLAGS_SHARED)) goto retry_private; @@ -2678,6 +2684,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, ret = -EWOULDBLOCK; } +out: return ret; } From ed7a061291a09d2dcdf5d9e02b1cc90888396b2a Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Mon, 2 Nov 2020 18:41:54 -0500 Subject: [PATCH 03/48] Revert "futex: Remove put_futex_key()" This reverts commit 9180bd467f9abdb44afde650d07e3b9dd66d837c. Signed-off-by: Kai Krakow --- kernel/futex.c | 61 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/kernel/futex.c b/kernel/futex.c index b978a76b8a5b1d..a1262c10c31567 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -660,6 +660,10 @@ static int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, return err; } +static inline void put_futex_key(union futex_key *key) +{ +} + /** * fault_in_user_writeable() - Fault in user address and verify RW access * @uaddr: pointer to faulting user space address @@ -1606,7 +1610,7 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) /* Make sure we really have tasks to wakeup */ if (!hb_waiters_pending(hb)) - goto out; + goto out_put_key; spin_lock(&hb->lock); @@ -1629,6 +1633,8 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) spin_unlock(&hb->lock); wake_up_q(&wake_q); +out_put_key: + put_futex_key(&key); out: return ret; } @@ -1699,7 +1705,7 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, goto out; ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE); if (unlikely(ret != 0)) - goto out; + goto out_put_key1; hb1 = hash_futex(&key1); hb2 = hash_futex(&key2); @@ -1717,13 +1723,13 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, * an MMU, but we might get them from range checking */ ret = op_ret; - goto out; + goto out_put_keys; } if (op_ret == -EFAULT) { ret = fault_in_user_writeable(uaddr2); if (ret) - goto out; + goto out_put_keys; } if (!(flags & FLAGS_SHARED)) { @@ -1731,6 +1737,8 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, goto retry_private; } + put_futex_key(&key2); + put_futex_key(&key1); cond_resched(); goto retry; } @@ -1766,6 +1774,10 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, out_unlock: double_unlock_hb(hb1, hb2); wake_up_q(&wake_q); +out_put_keys: + put_futex_key(&key2); +out_put_key1: + put_futex_key(&key1); out: return ret; } @@ -1977,7 +1989,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, requeue_pi ? FUTEX_WRITE : FUTEX_READ); if (unlikely(ret != 0)) - goto out; + goto out_put_key1; /* * The check above which compares uaddrs is not sufficient for @@ -1985,7 +1997,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, */ if (requeue_pi && match_futex(&key1, &key2)) { ret = -EINVAL; - goto out; + goto out_put_keys; } hb1 = hash_futex(&key1); @@ -2006,11 +2018,13 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, ret = get_user(curval, uaddr1); if (ret) - goto out; + goto out_put_keys; if (!(flags & FLAGS_SHARED)) goto retry_private; + put_futex_key(&key2); + put_futex_key(&key1); goto retry; } if (curval != *cmpval) { @@ -2069,6 +2083,8 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, case -EFAULT: double_unlock_hb(hb1, hb2); hb_waiters_dec(hb2); + put_futex_key(&key2); + put_futex_key(&key1); ret = fault_in_user_writeable(uaddr2); if (!ret) goto retry; @@ -2083,6 +2099,8 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, */ double_unlock_hb(hb1, hb2); hb_waiters_dec(hb2); + put_futex_key(&key2); + put_futex_key(&key1); /* * Handle the case where the owner is in the middle of * exiting. Wait for the exit to complete otherwise @@ -2192,6 +2210,10 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, wake_up_q(&wake_q); hb_waiters_dec(hb2); +out_put_keys: + put_futex_key(&key2); +out_put_key1: + put_futex_key(&key1); out: return ret ? ret : task_count; } @@ -2676,6 +2698,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, if (!(flags & FLAGS_SHARED)) goto retry_private; + put_futex_key(&q->key); goto retry; } @@ -2685,6 +2708,8 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, } out: + if (ret) + put_futex_key(&q->key); return ret; } @@ -2827,6 +2852,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, * - EAGAIN: The user space value changed. */ queue_unlock(hb); + put_futex_key(&q.key); /* * Handle the case where the owner is in the middle of * exiting. Wait for the exit to complete otherwise @@ -2919,11 +2945,13 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, /* Unqueue and drop the lock */ unqueue_me_pi(&q); - goto out; + goto out_put_key; out_unlock_put_key: queue_unlock(hb); +out_put_key: + put_futex_key(&q.key); out: if (to) { hrtimer_cancel(&to->timer); @@ -2936,11 +2964,12 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ret = fault_in_user_writeable(uaddr); if (ret) - goto out; + goto out_put_key; if (!(flags & FLAGS_SHARED)) goto retry_private; + put_futex_key(&q.key); goto retry; } @@ -3069,13 +3098,16 @@ static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) out_unlock: spin_unlock(&hb->lock); out_putkey: + put_futex_key(&key); return ret; pi_retry: + put_futex_key(&key); cond_resched(); goto retry; pi_faulted: + put_futex_key(&key); ret = fault_in_user_writeable(uaddr); if (!ret) @@ -3216,7 +3248,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, */ ret = futex_wait_setup(uaddr, val, flags, &q, &hb); if (ret) - goto out; + goto out_key2; /* * The check above which compares uaddrs is not sufficient for @@ -3225,7 +3257,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, if (match_futex(&q.key, &key2)) { queue_unlock(hb); ret = -EINVAL; - goto out; + goto out_put_keys; } /* Queue the futex_q, drop the hb lock, wait for wakeup. */ @@ -3235,7 +3267,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); spin_unlock(&hb->lock); if (ret) - goto out; + goto out_put_keys; /* * In order for us to be here, we know our q.key == key2, and since @@ -3311,6 +3343,11 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, ret = -EWOULDBLOCK; } +out_put_keys: + put_futex_key(&q.key); +out_key2: + put_futex_key(&key2); + out: if (to) { hrtimer_cancel(&to->timer); From 0db713a1c00003e87768cebbc0fa506397c2d0f6 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Fri, 12 Jul 2019 14:16:20 -0400 Subject: [PATCH 04/48] futex: Split key setup from key queue locking and read split the futex key setup from the queue locking and key reading. This is usefull to support the setup of multiple keys at the same time, like what is done in futex_requeue() and what will be done for the FUTEX_WAIT_MULTIPLE command. Signed-off-by: Gabriel Krisman Bertazi --- kernel/futex.c | 71 +++++++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/kernel/futex.c b/kernel/futex.c index a1262c10c31567..2a62425452770f 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -2637,6 +2637,39 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, __set_current_state(TASK_RUNNING); } +static int __futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, + struct futex_q *q, struct futex_hash_bucket **hb) +{ + + u32 uval; + int ret; + +retry_private: + *hb = queue_lock(q); + + ret = get_futex_value_locked(&uval, uaddr); + + if (ret) { + queue_unlock(*hb); + + ret = get_user(uval, uaddr); + if (ret) + return ret; + + if (!(flags & FLAGS_SHARED)) + goto retry_private; + + return 1; + } + + if (uval != val) { + queue_unlock(*hb); + ret = -EWOULDBLOCK; + } + + return ret; +} + /** * futex_wait_setup() - Prepare to wait on a futex * @uaddr: the futex userspace address @@ -2657,7 +2690,6 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, struct futex_q *q, struct futex_hash_bucket **hb) { - u32 uval; int ret; /* @@ -2678,38 +2710,19 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, * absorb a wakeup if *uaddr does not match the desired values * while the syscall executes. */ -retry: - ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ); - if (unlikely(ret != 0)) - return ret; - -retry_private: - *hb = queue_lock(q); + do { + ret = get_futex_key(uaddr, flags & FLAGS_SHARED, + &q->key, FUTEX_READ); + if (unlikely(ret != 0)) + return ret; - ret = get_futex_value_locked(&uval, uaddr); + ret = __futex_wait_setup(uaddr, val, flags, q, hb); - if (ret) { - queue_unlock(*hb); - - ret = get_user(uval, uaddr); + /* Drop key reference if retry or error. */ if (ret) - goto out; - - if (!(flags & FLAGS_SHARED)) - goto retry_private; - - put_futex_key(&q->key); - goto retry; - } + put_futex_key(&q->key); + } while (ret > 0); - if (uval != val) { - queue_unlock(*hb); - ret = -EWOULDBLOCK; - } - -out: - if (ret) - put_futex_key(&q->key); return ret; } From eed0263aac1f873b4fb7e3527b5e05f01f28603e Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Mon, 8 Jul 2019 09:44:09 -0400 Subject: [PATCH 05/48] futex: Implement FUTEX_WAIT_MULTIPLE This is a new futex operation to allow a thread to wait on several futexes at the same time, and wake up on any of them. In a sense, it implements one of the features that was supported by pooling on the old FUTEX_FD interface. My use case for this feature lies in Wine, where we want to implement a similar function available in Windows, mainly for event handling. The wine folks have an implementation of the userspace side using eventfd, but it suffers from bad performance, as shown in the measurements below. Technically, the old FUTEX_WAIT implementation can be easily reimplemented using do_futex_wait_multiple, with a count one, and I have a patch demonstrating how it works. I'm not proposing it, since futex is such a tricky code, that I'd be more confortable to have FUTEX_WAIT_MULTIPLE running upstream for a couple development cycles, before considering modifying FUTEX_WAIT. This was tested using three mechanisms: 1) By reimplementing FUTEX_WAIT in terms of FUTEX_WAIT_MULTIPLE and running tools/testing/selftests/futex and a full linux distro on top of this kernel. 2) By an example code that exercises the FUTEX_WAIT_MULTIPLE path on a multi thread, event handling setup. 3) By running the Wine fsync implementation and executing multi-threaded applications, in particular modern games on top of the implementation. Signed-off-by: Zebediah Figura Signed-off-by: Steven Noonan Signed-off-by: Pierre-Loup A. Griffais Signed-off-by: Gabriel Krisman Bertazi --- include/uapi/linux/futex.h | 7 ++ kernel/futex.c | 159 ++++++++++++++++++++++++++++++++++++- 2 files changed, 162 insertions(+), 4 deletions(-) diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index a89eb0accd5e2e..2401c4cf509545 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -21,6 +21,7 @@ #define FUTEX_WAKE_BITSET 10 #define FUTEX_WAIT_REQUEUE_PI 11 #define FUTEX_CMP_REQUEUE_PI 12 +#define FUTEX_WAIT_MULTIPLE 13 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 @@ -150,4 +151,10 @@ struct robust_list_head { (((op & 0xf) << 28) | ((cmp & 0xf) << 24) \ | ((oparg & 0xfff) << 12) | (cmparg & 0xfff)) +struct futex_wait_block { + __u32 __user *uaddr; + __u32 val; + __u32 bitset; +}; + #endif /* _UAPI_LINUX_FUTEX_H */ diff --git a/kernel/futex.c b/kernel/futex.c index 2a62425452770f..c1e918ba5e349c 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -165,6 +165,7 @@ static int __read_mostly futex_cmpxchg_enabled; #endif #define FLAGS_CLOCKRT 0x02 #define FLAGS_HAS_TIMEOUT 0x04 +#define FLAGS_WAKE_MULTIPLE 0x08 /* * Priority Inheritance state: @@ -2726,6 +2727,148 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, return ret; } +static int do_futex_wait_multiple(struct futex_wait_block *wb, + u32 count, unsigned int flags, + ktime_t *abs_time) +{ + + struct hrtimer_sleeper timeout, *to; + struct futex_hash_bucket *hb; + struct futex_q *qs = NULL; + int ret; + int i; + + qs = kcalloc(count, sizeof(struct futex_q), GFP_KERNEL); + if (!qs) + return -ENOMEM; + + to = futex_setup_timer(abs_time, &timeout, flags, + current->timer_slack_ns); + retry: + for (i = 0; i < count; i++) { + qs[i].key = FUTEX_KEY_INIT; + qs[i].bitset = wb[i].bitset; + + ret = get_futex_key(wb[i].uaddr, flags & FLAGS_SHARED, + &qs[i].key, FUTEX_READ); + if (unlikely(ret != 0)) { + for (--i; i >= 0; i--) + put_futex_key(&qs[i].key); + goto out; + } + } + + set_current_state(TASK_INTERRUPTIBLE); + + for (i = 0; i < count; i++) { + ret = __futex_wait_setup(wb[i].uaddr, wb[i].val, + flags, &qs[i], &hb); + if (ret) { + /* Drop the failed key directly. keys 0..(i-1) + * will be put by unqueue_me. */ + put_futex_key(&qs[i].key); + + /* Undo the partial work we did. */ + for (--i; i >= 0; i--) + unqueue_me(&qs[i]); + + __set_current_state(TASK_RUNNING); + if (ret > 0) + goto retry; + goto out; + } + + /* We can't hold to the bucket lock when dealing with + * the next futex. Queue ourselves now so we can unlock + * it before moving on. */ + queue_me(&qs[i], hb); + } + + if (to) + hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS); + + /* There is no easy to way to check if we are wake already on + * multiple futexes without waking through each one of them. So + * just sleep and let the scheduler handle it. + */ + if (!to || to->task) + freezable_schedule(); + + __set_current_state(TASK_RUNNING); + + ret = -ETIMEDOUT; + /* If we were woken (and unqueued), we succeeded. */ + for (i = 0; i < count; i++) + if (!unqueue_me(&qs[i])) + ret = i; + + /* Succeed wakeup */ + if (ret >= 0) + goto out; + + /* Woken by triggered timeout */ + if (to && !to->task) + goto out; + + /* + * We expect signal_pending(current), but we might be the + * victim of a spurious wakeup as well. + */ + if (!signal_pending(current)) + goto retry; + + ret = -ERESTARTSYS; + if (!abs_time) + goto out; + + ret = -ERESTART_RESTARTBLOCK; + out: + if (to) { + hrtimer_cancel(&to->timer); + destroy_hrtimer_on_stack(&to->timer); + } + + kfree(qs); + return ret; +} + +static int futex_wait_multiple(u32 __user *uaddr, unsigned int flags, + u32 count, ktime_t *abs_time) +{ + struct futex_wait_block *wb; + struct restart_block *restart; + int ret; + + if (!count) + return -EINVAL; + + wb = kcalloc(count, sizeof(struct futex_wait_block), GFP_KERNEL); + if (!wb) + return -ENOMEM; + + if (copy_from_user(wb, uaddr, + count * sizeof(struct futex_wait_block))) { + ret = -EFAULT; + goto out; + } + + ret = do_futex_wait_multiple(wb, count, flags, abs_time); + + if (ret == -ERESTART_RESTARTBLOCK) { + restart = ¤t->restart_block; + restart->fn = futex_wait_restart; + restart->futex.uaddr = uaddr; + restart->futex.val = count; + restart->futex.time = *abs_time; + restart->futex.flags = (flags | FLAGS_HAS_TIMEOUT | + FLAGS_WAKE_MULTIPLE); + } + +out: + kfree(wb); + return ret; +} + static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset) { @@ -2802,6 +2945,10 @@ static long futex_wait_restart(struct restart_block *restart) } restart->fn = do_no_restart_syscall; + if (restart->futex.flags & FLAGS_WAKE_MULTIPLE) + return (long)futex_wait_multiple(uaddr, restart->futex.flags, + restart->futex.val, tp); + return (long)futex_wait(uaddr, restart->futex.flags, restart->futex.val, tp, restart->futex.bitset); } @@ -3813,6 +3960,8 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, uaddr2); case FUTEX_CMP_REQUEUE_PI: return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1); + case FUTEX_WAIT_MULTIPLE: + return futex_wait_multiple(uaddr, flags, val, timeout); } return -ENOSYS; } @@ -3829,7 +3978,8 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || cmd == FUTEX_WAIT_BITSET || - cmd == FUTEX_WAIT_REQUEUE_PI)) { + cmd == FUTEX_WAIT_REQUEUE_PI || + cmd == FUTEX_WAIT_MULTIPLE)) { if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG)))) return -EFAULT; if (get_timespec64(&ts, utime)) @@ -3838,7 +3988,7 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, return -EINVAL; t = timespec64_to_ktime(ts); - if (cmd == FUTEX_WAIT) + if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE) t = ktime_add_safe(ktime_get(), t); else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) t = timens_ktime_to_host(CLOCK_MONOTONIC, t); @@ -4025,14 +4175,15 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || cmd == FUTEX_WAIT_BITSET || - cmd == FUTEX_WAIT_REQUEUE_PI)) { + cmd == FUTEX_WAIT_REQUEUE_PI || + cmd == FUTEX_WAIT_MULTIPLE)) { if (get_old_timespec32(&ts, utime)) return -EFAULT; if (!timespec64_valid(&ts)) return -EINVAL; t = timespec64_to_ktime(ts); - if (cmd == FUTEX_WAIT) + if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE) t = ktime_add_safe(ktime_get(), t); else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) t = timens_ktime_to_host(CLOCK_MONOTONIC, t); From 640dc2ec02e549d6d662edd8debf3c031c0b068d Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Mon, 2 Nov 2020 18:50:26 -0500 Subject: [PATCH 06/48] futex: Change WAIT_MULTIPLE opcode to 31 Signed-off-by: Gabriel Krisman Bertazi --- include/uapi/linux/futex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 2401c4cf509545..c34e52e0f787c8 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -21,7 +21,7 @@ #define FUTEX_WAKE_BITSET 10 #define FUTEX_WAIT_REQUEUE_PI 11 #define FUTEX_CMP_REQUEUE_PI 12 -#define FUTEX_WAIT_MULTIPLE 13 +#define FUTEX_WAIT_MULTIPLE 31 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 From 0ea2bd3d990933fb0efa9a91e3928758d291eb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:00 -0300 Subject: [PATCH 07/48] futex2: Implement wait and wake functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a new set of futex syscalls known as futex2. This new interface is aimed to implement a more maintainable code, while removing obsolete features and expanding it with new functionalities. Implements wait and wake semantics for futexes, along with the base infrastructure for future operations. The whole wait path is designed to be used by N waiters, thus making easier to implement vectorized wait. * Syscalls implemented by this patch: - futex_wait(void *uaddr, unsigned int val, unsigned int flags, struct timespec *timo) The user thread is put to sleep, waiting for a futex_wake() at uaddr, if the value at *uaddr is the same as val (otherwise, the syscall returns immediately with -EAGAIN). timo is an optional timeout value for the operation. Return 0 on success, error code otherwise. - futex_wake(void *uaddr, unsigned long nr_wake, unsigned int flags) Wake `nr_wake` threads waiting at uaddr. Return the number of woken threads on success, error code otherwise. ** The `flag` argument The flag is used to specify the size of the futex word (FUTEX_[8, 16, 32, 64]). It's mandatory to define one, since there's no default size. By default, the timeout uses a monotonic clock, but can be used as a realtime one by using the FUTEX_REALTIME_CLOCK flag. By default, futexes are of the private type, that means that this user address will be accessed by threads that shares the same memory region. This allows for some internal optimizations, so they are faster. However, if the address needs to be shared with different processes (like using `mmap()` or `shm()`), they need to be defined as shared and the flag FUTEX_SHARED_FLAG is used to set that. By default, the operation has no NUMA-awareness, meaning that the user can't choose the memory node where the kernel side futex data will be stored. The user can choose the node where it wants to operate by setting the FUTEX_NUMA_FLAG and using the following structure (where X can be 8, 16, or 32, 64): struct futexX_numa { __uX value; __sX hint; }; This structure should be passed at the `void *uaddr` of futex functions. The address of the structure will be used to be waited/waken on, and the `value` will be compared to `val` as usual. The `hint` member is used to defined which node the futex will use. When waiting, the futex will be registered on a kernel-side table stored on that node; when waking, the futex will be searched for on that given table. That means that there's no redundancy between tables, and the wrong `hint` value will led to undesired behavior. Userspace is responsible for dealing with node migrations issues that may occur. `hint` can range from [0, MAX_NUMA_NODES], for specifying a node, or -1, to use the same node the current process is using. When not using FUTEX_NUMA_FLAG on a NUMA system, the futex will be stored on a global table on some node, defined at compilation time. ** The `timo` argument As per the Y2038 work done in the kernel, new interfaces shouldn't add timeout options known to be buggy. Given that, `timo` should be a 64bit timeout at all platforms, using an absolute timeout value. Signed-off-by: André Almeida --- [RFC Add futex2 syscall 0/0] Hi, This patch series introduces the futex2 syscalls. * What happened to the current futex()? For some years now, developers have been trying to add new features to futex, but maintainers have been reluctant to accept then, given the multiplexed interface full of legacy features and tricky to do big changes. Some problems that people tried to address with patchsets are: NUMA-awareness[0], smaller sized futexes[1], wait on multiple futexes[2]. NUMA, for instance, just doesn't fit the current API in a reasonable way. Considering that, it's not possible to merge new features into the current futex. ** The NUMA problem At the current implementation, all futex kernel side infrastructure is stored on a single node. Given that, all futex() calls issued by processors that aren't located on that node will have a memory access penalty when doing it. ** The 32bit sized futex problem Embedded systems or anything with memory constrains would benefit of using smaller sizes for the futex userspace integer. Also, a mutex implementation can be done using just three values, so 8 bits is enough for various scenarios. ** The wait on multiple problem The use case lies in the Wine implementation of the Windows NT interface WaitMultipleObjects. This Windows API function allows a thread to sleep waiting on the first of a set of event sources (mutexes, timers, signal, console input, etc) to signal. Considering this is a primitive synchronization operation for Windows applications, being able to quickly signal events on the producer side, and quickly go to sleep on the consumer side is essential for good performance of those running over Wine. [0] https://lore.kernel.org/lkml/20160505204230.932454245@linutronix.de/ [1] https://lore.kernel.org/lkml/20191221155659.3159-2-malteskarupke@web.de/ [2] https://lore.kernel.org/lkml/20200213214525.183689-1-andrealmeid@collabora.com/ * The solution As proposed by Peter Zijlstra and Florian Weimer[3], a new interface is required to solve this, which must be designed with those features in mind. futex2() is that interface. As opposed to the current multiplexed interface, the new one should have one syscall per operation. This will allow the maintainability of the API if it gets extended, and will help users with type checking of arguments. In particular, the new interface is extended to support the ability to wait on any of a list of futexes at a time, which could be seen as a vectored extension of the FUTEX_WAIT semantics. [3] https://lore.kernel.org/lkml/20200303120050.GC2596@hirez.programming.kicks-ass.net/ * The interface The new interface can be seen in details in the following patches, but this is a high level summary of what the interface can do: - Supports wake/wait semantics, as in futex() - Supports requeue operations, similarly as FUTEX_CMP_REQUEUE, but with individual flags for each address - Supports waiting for a vector of futexes, using a new syscall named futex_waitv() - Supports variable sized futexes (8bits, 16bits, 32bits and 64bits) - Supports NUMA-awareness operations, where the user can specify on which memory node would like to operate * Implementation The internal implementation follows a similar design to the original futex. Given that we want to replicate the same external behavior of current futex, this should be somewhat expected. For some functions, like the init and the code to get a shared key, I literally copied code and comments from kernel/futex.c. I decided to do so instead of exposing the original function as a public function since in that way we can freely modify our implementation if required, without any impact on old futex. Also, the comments precisely describes the details and corner cases of the implementation. Each patch contains a brief description of implementation, but patch 6 "docs: locking: futex2: Add documentation" adds a more complete document about it. * The patchset This patchset can be also found at my git tree: https://gitlab.collabora.com/tonyk/linux/-/tree/futex2-dev - Patch 1: Implements wait/wake, and the basics foundations of futex2 - Patches 2-4: Implement the remaining features (shared, waitv, requeue). - Patch 5: Adds the x86_x32 ABI handling. I kept it in a separated patch since I'm not sure if x86_x32 is still a thing, or if it should return -ENOSYS. - Patch 6: Add a documentation file which details the interface and the internal implementation. - Patches 7-13: Selftests for all operations along with perf support for futex2. - Patch 14: While working on porting glibc for futex2, I found out that there's a futex_wake() call at the user thread exit path, if that thread was created with clone(..., CLONE_CHILD_SETTID, ...). In order to make pthreads work with futex2, it was required to add this patch. Note that this is more a proof-of-concept of what we will need to do in future, rather than part of the interface and shouldn't be merged as it is. * Testing: This patchset provides selftests for each operation and their flags. Along with that, the following work was done: ** Stability To stress the interface in "real world scenarios": - glibc[4]: nptl's low level locking was modified to use futex2 API (except for robust and PI things). All relevant nptl/ tests passed. - Wine[5]: Proton/Wine was modified in order to use futex2() for the emulation of Windows NT sync mechanisms based on futex, called "fsync". Triple-A games with huge CPU's loads and tons of parallel jobs worked as expected when compared with the previous FUTEX_WAIT_MULTIPLE implementation at futex(). Some games issue 42k futex2() calls per second. - Full GNU/Linux distro: I installed the modified glibc in my host machine, so all pthread's programs would use futex2(). After tweaking systemd[6] to allow futex2() calls at seccomp, everything worked as expected (web browsers do some syscall sandboxing and need some configuration as well). - perf: The perf benchmarks tests can also be used to stress the interface, and they can be found in this patchset. ** Performance - For comparing futex() and futex2() performance, I used the artificial benchmarks implemented at perf (wake, wake-parallel, hash and requeue). The setup was 200 runs for each test and using 8, 80, 800, 8000 for the number of threads, Note that for this test, I'm not using patch 14 ("kernel: Enable waitpid() for futex2") , for reasons explained at "The patchset" section. - For the first three ones, I measured an average of 4% gain in performance. This is not a big step, but it shows that the new interface is at least comparable in performance with the current one. - For requeue, I measured an average of 21% decrease in performance compared to the original futex implementation. This is expected given the new design with individual flags. The performance trade-offs are explained at patch 4 ("futex2: Implement requeue operation"). [4] https://gitlab.collabora.com/tonyk/glibc/-/tree/futex2 [5] https://gitlab.collabora.com/tonyk/wine/-/tree/proton_5.13 [6] https://gitlab.collabora.com/tonyk/systemd * FAQ ** "Where's the code for NUMA and FUTEX_8/16/64?" The current code is already complex enough to take some time for review, so I believe it's better to split that work out to a future iteration of this patchset. Besides that, this RFC is the core part of the infrastructure, and the following features will not pose big design changes to it, the work will be more about wiring up the flags and modifying some functions. ** "Where's the PI/robust stuff?" As said by Peter Zijlstra at [3], all those new features are related to the "simple" futex interface, that doesn't use PI or robust. Do we want to have this complexity at futex2() and if so, should it be part of this patchset or can it be future work? Thanks, André * Changelog Changes from v2: - API now supports 64bit futexes, in addition to 8, 16 and 32. - Refactored futex2_wait and futex2_waitv selftests v2: https://lore.kernel.org/lkml/20210304004219.134051-1-andrealmeid@collabora.com/ Changes from v1: - Unified futex_set_timer_and_wait and __futex_wait code - Dropped _carefull from linked list function calls - Fixed typos on docs patch - uAPI flags are now added as features are introduced, instead of all flags in patch 1 - Removed struct futex_single_waiter in favor of an anon struct v1: https://lore.kernel.org/lkml/20210215152404.250281-1-andrealmeid@collabora.com/ --- MAINTAINERS | 2 +- arch/arm/tools/syscall.tbl | 2 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 4 + arch/x86/entry/syscalls/syscall_32.tbl | 2 + arch/x86/entry/syscalls/syscall_64.tbl | 2 + include/linux/compat.h | 12 + include/linux/syscalls.h | 6 + include/uapi/asm-generic/unistd.h | 8 +- include/uapi/linux/futex.h | 5 + init/Kconfig | 7 + kernel/Makefile | 1 + kernel/futex2.c | 610 ++++++++++++++++++ kernel/sys_ni.c | 4 + tools/include/uapi/asm-generic/unistd.h | 8 +- .../arch/x86/entry/syscalls/syscall_64.tbl | 2 + 16 files changed, 673 insertions(+), 4 deletions(-) create mode 100644 kernel/futex2.c diff --git a/MAINTAINERS b/MAINTAINERS index 4fef10dd297533..1757eb0d65a180 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7280,7 +7280,7 @@ F: Documentation/locking/*futex* F: include/asm-generic/futex.h F: include/linux/futex.h F: include/uapi/linux/futex.h -F: kernel/futex.c +F: kernel/futex* F: tools/perf/bench/futex* F: tools/testing/selftests/futex/ diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index d056a548358ea0..9ac7edbb70b659 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -454,3 +454,5 @@ 438 common pidfd_getfd sys_pidfd_getfd 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise +441 common futex_wait sys_futex_wait +442 common futex_wake sys_futex_wake diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index b3b2019f8d16bb..949788f5ba4007 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 441 +#define __NR_compat_syscalls 443 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 107f08e03b9fdc..df008993b00fda 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -889,6 +889,10 @@ __SYSCALL(__NR_pidfd_getfd, sys_pidfd_getfd) __SYSCALL(__NR_faccessat2, sys_faccessat2) #define __NR_process_madvise 440 __SYSCALL(__NR_process_madvise, sys_process_madvise) +#define __NR_futex_wait 441 +__SYSCALL(__NR_futex_wait, sys_futex_wait) +#define __NR_futex_wake 442 +__SYSCALL(__NR_futex_wake, sys_futex_wake) /* * Please add new compat syscalls above this comment and update diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 0d0667a9fbd70a..83a75ff39b1f3b 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -445,3 +445,5 @@ 438 i386 pidfd_getfd sys_pidfd_getfd 439 i386 faccessat2 sys_faccessat2 440 i386 process_madvise sys_process_madvise +441 i386 futex_wait sys_futex_wait +442 i386 futex_wake sys_futex_wake diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 379819244b91d2..6658fd63c7b897 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -362,6 +362,8 @@ 438 common pidfd_getfd sys_pidfd_getfd 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise +441 common futex_wait sys_futex_wait +442 common futex_wake sys_futex_wake # # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/compat.h b/include/linux/compat.h index 14d514233e1d40..e63005233f2763 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -616,6 +616,18 @@ asmlinkage long compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr, compat_size_t __user *len_ptr); +/* kernel/futex2.c */ +asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, + compat_uint_t nr_futexes, compat_uint_t flags, + struct __kernel_timespec __user *timo); + +asmlinkage long compat_sys_futex_requeue(struct compat_futex_requeue *uaddr1, + struct compat_futex_requeue *uaddr2, + compat_uint_t nr_wake, + compat_uint_t nr_requeue, + compat_u64 cmpval, + compat_uint_t flags); + /* kernel/itimer.c */ asmlinkage long compat_sys_getitimer(int which, struct old_itimerval32 __user *it); diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index aea0ce9f3b745a..c03bc7da78c075 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -613,6 +613,12 @@ asmlinkage long sys_get_robust_list(int pid, asmlinkage long sys_set_robust_list(struct robust_list_head __user *head, size_t len); +/* kernel/futex2.c */ +asmlinkage long sys_futex_wait(void __user *uaddr, unsigned int val, unsigned int flags, + struct __kernel_timespec __user *timo); +asmlinkage long sys_futex_wake(void __user *uaddr, unsigned int nr_wake, + unsigned int flags); + /* kernel/hrtimer.c */ asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp, struct __kernel_timespec __user *rmtp); diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 2056318988f774..ae47d6a9e9d244 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -860,8 +860,14 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) #define __NR_process_madvise 440 __SYSCALL(__NR_process_madvise, sys_process_madvise) +#define __NR_futex_wait 441 +__SYSCALL(__NR_futex_wait, sys_futex_wait) + +#define __NR_futex_wake 442 +__SYSCALL(__NR_futex_wake, sys_futex_wake) + #undef __NR_syscalls -#define __NR_syscalls 441 +#define __NR_syscalls 443 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index c34e52e0f787c8..eb7aab802aec8e 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -42,6 +42,11 @@ #define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | \ FUTEX_PRIVATE_FLAG) +/* Size argument to futex2 syscall */ +#define FUTEX_32 2 + +#define FUTEX_SIZE_MASK 0x3 + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/init/Kconfig b/init/Kconfig index fc4c9f416fadbb..1399d6d24bd6a7 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1532,6 +1532,13 @@ config FUTEX support for "fast userspace mutexes". The resulting kernel may not run glibc-based applications correctly. +config FUTEX2 + bool "Enable futex2 support" if EXPERT + depends on FUTEX + default y + help + Support for futex2 interface. + config FUTEX_PI bool depends on FUTEX && RT_MUTEXES diff --git a/kernel/Makefile b/kernel/Makefile index e7905bdf6e972c..f2cbf9f98498f7 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -54,6 +54,7 @@ obj-$(CONFIG_PROFILING) += profile.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-y += time/ obj-$(CONFIG_FUTEX) += futex.o +obj-$(CONFIG_FUTEX2) += futex2.o obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o obj-$(CONFIG_SMP) += smp.o ifneq ($(CONFIG_SMP),y) diff --git a/kernel/futex2.c b/kernel/futex2.c new file mode 100644 index 00000000000000..7d8aec4644d7c2 --- /dev/null +++ b/kernel/futex2.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * futex2 system call interface by André Almeida + * + * Copyright 2021 Collabora Ltd. + * + * Based on original futex implementation by: + * (C) 2002 Rusty Russell, IBM + * (C) 2003, 2006 Ingo Molnar, Red Hat Inc. + * (C) 2003, 2004 Jamie Lokier + * (C) 2006 Thomas Gleixner, Timesys Corp. + * (C) 2007 Eric Dumazet + * (C) 2009 Darren Hart, IBM + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * struct futex_key - Components to build unique key for a futex + * @pointer: Pointer to current->mm + * @index: Start address of the page containing futex + * @offset: Address offset of uaddr in a page + */ +struct futex_key { + u64 pointer; + unsigned long index; + unsigned long offset; +}; + +/** + * struct futex_waiter - List entry for a waiter + * @uaddr: Virtual address of userspace futex + * @key: Information that uniquely identify a futex + * @list: List node struct + * @val: Expected value for this waiter + * @flags: Flags + * @bucket: Pointer to the bucket for this waiter + * @index: Index of waiter in futexv list + */ +struct futex_waiter { + void __user *uaddr; + struct futex_key key; + struct list_head list; + u64 val; + unsigned int flags; + struct futex_bucket *bucket; + unsigned int index; +}; + +/** + * struct futex_waiter_head - List of futexes to be waited + * @task: Task to be awaken + * @hint: Was someone on this list awakened? + * @objects: List of futexes + */ +struct futex_waiter_head { + struct task_struct *task; + bool hint; + struct futex_waiter objects[0]; +}; + +/** + * struct futex_bucket - A bucket of futex's hash table + * @waiters: Number of waiters in the bucket + * @lock: Bucket lock + * @list: List of waiters on this bucket + */ +struct futex_bucket { + atomic_t waiters; + spinlock_t lock; + struct list_head list; +}; + +/* Mask for futex2 flag operations */ +#define FUTEX2_MASK (FUTEX_SIZE_MASK | FUTEX_CLOCK_REALTIME) + +static struct futex_bucket *futex_table; +static unsigned int futex2_hashsize; + +/* + * Reflects a new waiter being added to the waitqueue. + */ +static inline void bucket_inc_waiters(struct futex_bucket *bucket) +{ +#ifdef CONFIG_SMP + atomic_inc(&bucket->waiters); + /* + * Issue a barrier after adding so futex_wake() will see that the + * value had increased + */ + smp_mb__after_atomic(); +#endif +} + +/* + * Reflects a waiter being removed from the waitqueue by wakeup + * paths. + */ +static inline void bucket_dec_waiters(struct futex_bucket *bucket) +{ +#ifdef CONFIG_SMP + atomic_dec(&bucket->waiters); +#endif +} + +/* + * Get the number of waiters in a bucket + */ +static inline int bucket_get_waiters(struct futex_bucket *bucket) +{ +#ifdef CONFIG_SMP + /* + * Issue a barrier before reading so we get an updated value from + * futex_wait() + */ + smp_mb(); + return atomic_read(&bucket->waiters); +#else + return 1; +#endif +} + +/** + * futex_get_bucket - Check if the user address is valid, prepare internal + * data and calculate the hash + * @uaddr: futex user address + * @key: data that uniquely identifies a futex + * + * Return: address of bucket on success, error code otherwise + */ +static struct futex_bucket *futex_get_bucket(void __user *uaddr, + struct futex_key *key) +{ + uintptr_t address = (uintptr_t)uaddr; + u32 hash_key; + + /* Checking if uaddr is valid and accessible */ + if (unlikely(!IS_ALIGNED(address, sizeof(u32)))) + return ERR_PTR(-EINVAL); + if (unlikely(!access_ok(uaddr, sizeof(u32)))) + return ERR_PTR(-EFAULT); + + key->offset = address % PAGE_SIZE; + address -= key->offset; + key->pointer = (u64)address; + key->index = (unsigned long)current->mm; + + /* Generate hash key for this futex using uaddr and current->mm */ + hash_key = jhash2((u32 *)key, sizeof(*key) / sizeof(u32), 0); + + /* Since HASH_SIZE is 2^n, subtracting 1 makes a perfect bit mask */ + return &futex_table[hash_key & (futex2_hashsize - 1)]; +} + +/** + * futex_get_user - Get the userspace value on this address + * @uval: variable to store the value + * @uaddr: userspace address + * + * Check the comment at futex_enqueue() for more information. + */ +static int futex_get_user(u32 *uval, u32 __user *uaddr) +{ + int ret; + + pagefault_disable(); + ret = __get_user(*uval, uaddr); + pagefault_enable(); + + return ret; +} + +/** + * futex_setup_time - Prepare the timeout mechanism and start it. + * @timo: Timeout value from userspace + * @timeout: Pointer to hrtimer handler + * @flags: Flags from userspace, to decide which clockid to use + * + * Return: 0 on success, error code otherwise + */ +static int futex_setup_time(struct __kernel_timespec __user *timo, + struct hrtimer_sleeper *timeout, + unsigned int flags) +{ + ktime_t time; + struct timespec64 ts; + clockid_t clockid = (flags & FUTEX_CLOCK_REALTIME) ? + CLOCK_REALTIME : CLOCK_MONOTONIC; + + if (get_timespec64(&ts, timo)) + return -EFAULT; + + if (!timespec64_valid(&ts)) + return -EINVAL; + + time = timespec64_to_ktime(ts); + + hrtimer_init_sleeper(timeout, clockid, HRTIMER_MODE_ABS); + + hrtimer_set_expires(&timeout->timer, time); + + hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS); + + return 0; +} + +/** + * futex_dequeue_multiple - Remove multiple futexes from hash table + * @futexv: list of waiters + * @nr: number of futexes to be removed + * + * This function is used if (a) something went wrong while enqueuing, and we + * need to undo our work (then nr <= nr_futexes) or (b) we woke up, and thus + * need to remove every waiter, check if some was indeed woken and return. + * Before removing a waiter, we check if it's on the list, since we have no + * clue who have been waken. + * + * Return: + * * -1 - If no futex was woken during the removal + * * 0>= - At least one futex was found woken, index of the last one + */ +static int futex_dequeue_multiple(struct futex_waiter_head *futexv, unsigned int nr) +{ + int i, ret = -1; + + for (i = 0; i < nr; i++) { + spin_lock(&futexv->objects[i].bucket->lock); + if (!list_empty(&futexv->objects[i].list)) { + list_del_init(&futexv->objects[i].list); + bucket_dec_waiters(futexv->objects[i].bucket); + } else { + ret = i; + } + spin_unlock(&futexv->objects[i].bucket->lock); + } + + return ret; +} + +/** + * futex_enqueue - Check the value and enqueue a futex on a wait list + * + * @futexv: List of futexes + * @nr_futexes: Number of futexes in the list + * @awakened: If a futex was awakened during enqueueing, store the index here + * + * Get the value from the userspace address and compares with the expected one. + * + * Getting the value from user futex address: + * + * Since we are in a hurry, we use a spin lock and we can't sleep. + * Try to get the value with page fault disabled (when enable, we might + * sleep). + * + * If we fail, we aren't sure if the address is invalid or is just a + * page fault. Then, release the lock (so we can sleep) and try to get + * the value with page fault enabled. In order to trigger a page fault + * handling, we just call __get_user() again. If we sleep with enqueued + * futexes, we might miss a wake, so dequeue everything before sleeping. + * + * If get_user succeeds, this mean that the address is valid and we do + * the work again. Since we just handled the page fault, the page is + * likely pinned in memory and we should be luckier this time and be + * able to get the value. If we fail anyway, we will try again. + * + * If even with page faults enabled we get and error, this means that + * the address is not valid and we return from the syscall. + * + * If we got an unexpected value or need to treat a page fault and realized that + * a futex was awakened, we can priority this and return success. + * + * In success, enqueue the futex in the correct bucket + * + * Return: + * * 1 - We were awake in the process and nothing is enqueued + * * 0 - Everything is enqueued and we are ready to sleep + * * 0< - Something went wrong, nothing is enqueued, return error code + */ +static int futex_enqueue(struct futex_waiter_head *futexv, unsigned int nr_futexes, + int *awakened) +{ + int i, ret; + u32 uval, val; + u32 __user *uaddr; + struct futex_bucket *bucket; + +retry: + set_current_state(TASK_INTERRUPTIBLE); + + for (i = 0; i < nr_futexes; i++) { + uaddr = (u32 __user *)futexv->objects[i].uaddr; + val = (u32)futexv->objects[i].val; + + bucket = futexv->objects[i].bucket; + + bucket_inc_waiters(bucket); + spin_lock(&bucket->lock); + + ret = futex_get_user(&uval, uaddr); + + if (unlikely(ret)) { + spin_unlock(&bucket->lock); + + bucket_dec_waiters(bucket); + __set_current_state(TASK_RUNNING); + *awakened = futex_dequeue_multiple(futexv, i); + + if (*awakened >= 0) + return 1; + + if (__get_user(uval, uaddr)) + return -EFAULT; + + goto retry; + } + + if (uval != val) { + spin_unlock(&bucket->lock); + + bucket_dec_waiters(bucket); + __set_current_state(TASK_RUNNING); + *awakened = futex_dequeue_multiple(futexv, i); + + if (*awakened >= 0) + return 1; + + return -EAGAIN; + } + + list_add_tail(&futexv->objects[i].list, &bucket->list); + spin_unlock(&bucket->lock); + } + + return 0; +} + +/** + * __futex_waitv - Enqueue the list of futexes and wait to be woken + * @futexv: List of futexes to wait + * @nr_futexes: Length of futexv + * @timo: Timeout + * @flags: Timeout flags + * + * Return: + * * 0 >= - Hint of which futex woke us + * * 0 < - Error code + */ +static int __futex_waitv(struct futex_waiter_head *futexv, unsigned int nr_futexes, + struct __kernel_timespec __user *timo, + unsigned int flags) +{ + int ret; + struct hrtimer_sleeper timeout; + + if (timo) { + ret = futex_setup_time(timo, &timeout, flags); + if (ret) + return ret; + } + + while (1) { + int awakened = -1; + + ret = futex_enqueue(futexv, nr_futexes, &awakened); + + if (ret) { + if (awakened >= 0) + ret = awakened; + break; + } + + /* Before sleeping, check if someone was woken */ + if (!futexv->hint && (!timo || timeout.task)) + freezable_schedule(); + + __set_current_state(TASK_RUNNING); + + /* + * One of those things triggered this wake: + * + * * We have been removed from the bucket. futex_wake() woke + * us. We just need to dequeue and return 0 to userspace. + * + * However, if no futex was dequeued by a futex_wake(): + * + * * If the there's a timeout and it has expired, + * return -ETIMEDOUT. + * + * * If there is a signal pending, something wants to kill our + * thread, return -ERESTARTSYS. + * + * * If there's no signal pending, it was a spurious wake + * (scheduler gave us a chance to do some work, even if we + * don't want to). We need to remove ourselves from the + * bucket and add again, to prevent losing wakeups in the + * meantime. + */ + + ret = futex_dequeue_multiple(futexv, nr_futexes); + + /* Normal wake */ + if (ret >= 0) + break; + + if (timo && !timeout.task) { + ret = -ETIMEDOUT; + break; + } + + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + /* Spurious wake, do everything again */ + } + + if (timo) + hrtimer_cancel(&timeout.timer); + + return ret; +} + +static long ksys_futex_wait(void __user *uaddr, unsigned int val, unsigned int flags, + struct __kernel_timespec __user *timo) +{ + unsigned int size = flags & FUTEX_SIZE_MASK; + struct futex_waiter *waiter; + struct futex_waiter_head *futexv; + + /* Wrapper for a futexv_waiter_head with one element */ + struct { + struct futex_waiter_head futexv; + struct futex_waiter waiter; + } __packed wait_single; + + if (flags & ~FUTEX2_MASK) + return -EINVAL; + + if (size != FUTEX_32) + return -EINVAL; + + futexv = &wait_single.futexv; + futexv->task = current; + futexv->hint = false; + + waiter = &wait_single.waiter; + waiter->index = 0; + waiter->val = val; + waiter->uaddr = uaddr; + memset(&wait_single.waiter.key, 0, sizeof(struct futex_key)); + + INIT_LIST_HEAD(&waiter->list); + + /* Get an unlocked hash bucket */ + waiter->bucket = futex_get_bucket(uaddr, &waiter->key); + if (IS_ERR(waiter->bucket)) + return PTR_ERR(waiter->bucket); + + return __futex_waitv(futexv, 1, timo, flags); +} + +/** + * sys_futex_wait - Wait on a futex address if (*uaddr) == val + * @uaddr: User address of futex + * @val: Expected value of futex + * @flags: Specify the size of futex and the clockid + * @timo: Optional absolute timeout. + * + * The user thread is put to sleep, waiting for a futex_wake() at uaddr, if the + * value at *uaddr is the same as val (otherwise, the syscall returns + * immediately with -EAGAIN). + * + * Returns 0 on success, error code otherwise. + */ +SYSCALL_DEFINE4(futex_wait, void __user *, uaddr, unsigned int, val, unsigned int, flags, + struct __kernel_timespec __user *, timo) +{ + + return ksys_futex_wait(uaddr, val, flags, timo); +} + +/** + * futex_get_parent - For a given futex in a futexv list, get a pointer to the futexv + * @waiter: Address of futex in the list + * @index: Index of futex in the list + * + * Return: A pointer to its futexv struct + */ +static inline struct futex_waiter_head *futex_get_parent(uintptr_t waiter, + unsigned int index) +{ + uintptr_t parent = waiter - sizeof(struct futex_waiter_head) + - (uintptr_t)(index * sizeof(struct futex_waiter)); + + return (struct futex_waiter_head *)parent; +} + +/** + * futex_mark_wake - Find the task to be wake and add it in wake queue + * @waiter: Waiter to be wake + * @bucket: Bucket to be decremented + * @wake_q: Wake queue to insert the task + */ +static void futex_mark_wake(struct futex_waiter *waiter, + struct futex_bucket *bucket, + struct wake_q_head *wake_q) +{ + struct task_struct *task; + struct futex_waiter_head *parent = futex_get_parent((uintptr_t)waiter, + waiter->index); + + lockdep_assert_held(&bucket->lock); + parent->hint = true; + task = parent->task; + get_task_struct(task); + list_del_init(&waiter->list); + wake_q_add_safe(wake_q, task); + bucket_dec_waiters(bucket); +} + +static inline bool futex_match(struct futex_key key1, struct futex_key key2) +{ + return (key1.index == key2.index && + key1.pointer == key2.pointer && + key1.offset == key2.offset); +} + +/** + * sys_futex_wake - Wake a number of futexes waiting on an address + * @uaddr: Address of futex to be woken up + * @nr_wake: Number of futexes waiting in uaddr to be woken up + * @flags: Flags for size and shared + * + * Wake `nr_wake` threads waiting at uaddr. + * + * Returns the number of woken threads on success, error code otherwise. + */ +SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, + unsigned int, flags) +{ + unsigned int size = flags & FUTEX_SIZE_MASK; + struct futex_waiter waiter, *aux, *tmp; + struct futex_bucket *bucket; + DEFINE_WAKE_Q(wake_q); + int ret = 0; + + if (flags & ~FUTEX2_MASK) + return -EINVAL; + + if (size != FUTEX_32) + return -EINVAL; + + bucket = futex_get_bucket(uaddr, &waiter.key); + if (IS_ERR(bucket)) + return PTR_ERR(bucket); + + if (!bucket_get_waiters(bucket) || !nr_wake) + return 0; + + spin_lock(&bucket->lock); + list_for_each_entry_safe(aux, tmp, &bucket->list, list) { + if (futex_match(waiter.key, aux->key)) { + futex_mark_wake(aux, bucket, &wake_q); + if (++ret >= nr_wake) + break; + } + } + spin_unlock(&bucket->lock); + + wake_up_q(&wake_q); + + return ret; +} + +static int __init futex2_init(void) +{ + int i; + unsigned int futex_shift; + +#if CONFIG_BASE_SMALL + futex2_hashsize = 16; +#else + futex2_hashsize = roundup_pow_of_two(256 * num_possible_cpus()); +#endif + + futex_table = alloc_large_system_hash("futex2", sizeof(struct futex_bucket), + futex2_hashsize, 0, + futex2_hashsize < 256 ? HASH_SMALL : 0, + &futex_shift, NULL, + futex2_hashsize, futex2_hashsize); + futex2_hashsize = 1UL << futex_shift; + + BUG_ON(!is_power_of_2(futex2_hashsize)); + + for (i = 0; i < futex2_hashsize; i++) { + INIT_LIST_HEAD(&futex_table[i].list); + spin_lock_init(&futex_table[i].lock); + atomic_set(&futex_table[i].waiters, 0); + } + + return 0; +} +core_initcall(futex2_init); diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index f27ac94d5fa727..35ff743b17d0ec 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -148,6 +148,10 @@ COND_SYSCALL_COMPAT(set_robust_list); COND_SYSCALL(get_robust_list); COND_SYSCALL_COMPAT(get_robust_list); +/* kernel/futex2.c */ +COND_SYSCALL(futex_wait); +COND_SYSCALL(futex_wake); + /* kernel/hrtimer.c */ /* kernel/itimer.c */ diff --git a/tools/include/uapi/asm-generic/unistd.h b/tools/include/uapi/asm-generic/unistd.h index 2056318988f774..ae47d6a9e9d244 100644 --- a/tools/include/uapi/asm-generic/unistd.h +++ b/tools/include/uapi/asm-generic/unistd.h @@ -860,8 +860,14 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) #define __NR_process_madvise 440 __SYSCALL(__NR_process_madvise, sys_process_madvise) +#define __NR_futex_wait 441 +__SYSCALL(__NR_futex_wait, sys_futex_wait) + +#define __NR_futex_wake 442 +__SYSCALL(__NR_futex_wake, sys_futex_wake) + #undef __NR_syscalls -#define __NR_syscalls 441 +#define __NR_syscalls 443 /* * 32 bit systems traditionally used different diff --git a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl index 379819244b91d2..47de3bf93fe6f9 100644 --- a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl +++ b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl @@ -362,6 +362,8 @@ 438 common pidfd_getfd sys_pidfd_getfd 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise +441 common futex_wait sys_futex_wait +442 common futex_wake sys_futex_wake # # Due to a historical design error, certain syscalls are numbered differently From 9e238f5820fdbca67911ce99fdc0324cf5395880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:01 -0300 Subject: [PATCH 08/48] futex2: Add support for shared futexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for shared futexes for cross-process resources. This design relies on the same approach done in old futex to create an unique id for file-backed shared memory, by using a counter at struct inode. There are two types of futexes: private and shared ones. The private are futexes meant to be used by threads that shares the same memory space, are easier to be uniquely identified an thus can have some performance optimization. The elements for identifying one are: the start address of the page where the address is, the address offset within the page and the current->mm pointer. Now, for uniquely identifying shared futex: - If the page containing the user address is an anonymous page, we can just use the same data used for private futexes (the start address of the page, the address offset within the page and the current->mm pointer) that will be enough for uniquely identifying such futex. We also set one bit at the key to differentiate if a private futex is used on the same address (mixing shared and private calls are not allowed). - If the page is file-backed, current->mm maybe isn't the same one for every user of this futex, so we need to use other data: the page->index, an UUID for the struct inode and the offset within the page. Note that members of futex_key doesn't have any particular meaning after they are part of the struct - they are just bytes to identify a futex. Given that, we don't need to use a particular name or type that matches the original data, we only need to care about the bitsize of each component and make both private and shared data fit in the same memory space. Signed-off-by: André Almeida --- fs/inode.c | 1 + include/linux/fs.h | 1 + include/uapi/linux/futex.h | 2 + kernel/futex2.c | 221 +++++++++++++++++++++++++++++++++++-- 4 files changed, 218 insertions(+), 7 deletions(-) diff --git a/fs/inode.c b/fs/inode.c index 5eea9912a0b9d8..66673dfb398dea 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -139,6 +139,7 @@ int inode_init_always(struct super_block *sb, struct inode *inode) inode->i_blkbits = sb->s_blocksize_bits; inode->i_flags = 0; atomic64_set(&inode->i_sequence, 0); + atomic64_set(&inode->i_sequence2, 0); atomic_set(&inode->i_count, 1); inode->i_op = &empty_iops; inode->i_fop = &no_open_fops; diff --git a/include/linux/fs.h b/include/linux/fs.h index 8bde32cf971159..2216978cda6361 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -681,6 +681,7 @@ struct inode { }; atomic64_t i_version; atomic64_t i_sequence; /* see futex */ + atomic64_t i_sequence2; /* see futex2 */ atomic_t i_count; atomic_t i_dio_count; atomic_t i_writecount; diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index eb7aab802aec8e..197bd5d329ebbb 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -47,6 +47,8 @@ #define FUTEX_SIZE_MASK 0x3 +#define FUTEX_SHARED_FLAG 8 + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/kernel/futex2.c b/kernel/futex2.c index 7d8aec4644d7c2..a1a31f3e41f6fe 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -14,8 +14,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -23,8 +25,8 @@ /** * struct futex_key - Components to build unique key for a futex - * @pointer: Pointer to current->mm - * @index: Start address of the page containing futex + * @pointer: Pointer to current->mm or inode's UUID for file backed futexes + * @index: Start address of the page containing futex or index of the page * @offset: Address offset of uaddr in a page */ struct futex_key { @@ -78,7 +80,12 @@ struct futex_bucket { }; /* Mask for futex2 flag operations */ -#define FUTEX2_MASK (FUTEX_SIZE_MASK | FUTEX_CLOCK_REALTIME) +#define FUTEX2_MASK (FUTEX_SIZE_MASK | FUTEX_CLOCK_REALTIME | FUTEX_SHARED_FLAG) + +#define is_object_shared ((futexv->objects[i].flags & FUTEX_SHARED_FLAG) ? true : false) + +#define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ +#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */ static struct futex_bucket *futex_table; static unsigned int futex2_hashsize; @@ -126,16 +133,198 @@ static inline int bucket_get_waiters(struct futex_bucket *bucket) #endif } +/** + * futex_get_inode_uuid - Gets an UUID for an inode + * @inode: inode to get UUID + * + * Generate a machine wide unique identifier for this inode. + * + * This relies on u64 not wrapping in the life-time of the machine; which with + * 1ns resolution means almost 585 years. + * + * This further relies on the fact that a well formed program will not unmap + * the file while it has a (shared) futex waiting on it. This mapping will have + * a file reference which pins the mount and inode. + * + * If for some reason an inode gets evicted and read back in again, it will get + * a new sequence number and will _NOT_ match, even though it is the exact same + * file. + * + * It is important that match_futex() will never have a false-positive, esp. + * for PI futexes that can mess up the state. The above argues that false-negatives + * are only possible for malformed programs. + * + * Returns: UUID for the given inode + */ +static u64 futex_get_inode_uuid(struct inode *inode) +{ + static atomic64_t i_seq; + u64 old; + + /* Does the inode already have a sequence number? */ + old = atomic64_read(&inode->i_sequence2); + + if (likely(old)) + return old; + + for (;;) { + u64 new = atomic64_add_return(1, &i_seq); + + if (WARN_ON_ONCE(!new)) + continue; + + old = atomic64_cmpxchg_relaxed(&inode->i_sequence2, 0, new); + if (old) + return old; + return new; + } +} + +/** + * futex_get_shared_key - Get a key for a shared futex + * @address: Futex memory address + * @mm: Current process mm_struct pointer + * @key: Key struct to be filled + * + * Returns: 0 on success, error code otherwise + */ +static int futex_get_shared_key(uintptr_t address, struct mm_struct *mm, + struct futex_key *key) +{ + int ret; + struct page *page, *tail; + struct address_space *mapping; + +again: + ret = get_user_pages_fast(address, 1, 0, &page); + if (ret < 0) + return ret; + + /* + * The treatment of mapping from this point on is critical. The page + * lock protects many things but in this context the page lock + * stabilizes mapping, prevents inode freeing in the shared + * file-backed region case and guards against movement to swap cache. + * + * Strictly speaking the page lock is not needed in all cases being + * considered here and page lock forces unnecessarily serialization + * From this point on, mapping will be re-verified if necessary and + * page lock will be acquired only if it is unavoidable + * + * Mapping checks require the head page for any compound page so the + * head page and mapping is looked up now. For anonymous pages, it + * does not matter if the page splits in the future as the key is + * based on the address. For filesystem-backed pages, the tail is + * required as the index of the page determines the key. For + * base pages, there is no tail page and tail == page. + */ + tail = page; + page = compound_head(page); + mapping = READ_ONCE(page->mapping); + + /* + * If page->mapping is NULL, then it cannot be a PageAnon + * page; but it might be the ZERO_PAGE or in the gate area or + * in a special mapping (all cases which we are happy to fail); + * or it may have been a good file page when get_user_pages_fast + * found it, but truncated or holepunched or subjected to + * invalidate_complete_page2 before we got the page lock (also + * cases which we are happy to fail). And we hold a reference, + * so refcount care in invalidate_complete_page's remove_mapping + * prevents drop_caches from setting mapping to NULL beneath us. + * + * The case we do have to guard against is when memory pressure made + * shmem_writepage move it from filecache to swapcache beneath us: + * an unlikely race, but we do need to retry for page->mapping. + */ + if (unlikely(!mapping)) { + int shmem_swizzled; + + /* + * Page lock is required to identify which special case above + * applies. If this is really a shmem page then the page lock + * will prevent unexpected transitions. + */ + lock_page(page); + shmem_swizzled = PageSwapCache(page) || page->mapping; + unlock_page(page); + put_page(page); + + if (shmem_swizzled) + goto again; + + return -EFAULT; + } + + /* + * If the futex key is stored on an anonymous page, then the associated + * object is the mm which is implicitly pinned by the calling process. + * + * NOTE: When userspace waits on a MAP_SHARED mapping, even if + * it's a read-only handle, it's expected that futexes attach to + * the object not the particular process. + */ + if (PageAnon(page)) { + key->offset |= FUT_OFF_MMSHARED; + } else { + struct inode *inode; + + /* + * The associated futex object in this case is the inode and + * the page->mapping must be traversed. Ordinarily this should + * be stabilised under page lock but it's not strictly + * necessary in this case as we just want to pin the inode, not + * update the radix tree or anything like that. + * + * The RCU read lock is taken as the inode is finally freed + * under RCU. If the mapping still matches expectations then the + * mapping->host can be safely accessed as being a valid inode. + */ + rcu_read_lock(); + + if (READ_ONCE(page->mapping) != mapping) { + rcu_read_unlock(); + put_page(page); + + goto again; + } + + inode = READ_ONCE(mapping->host); + if (!inode) { + rcu_read_unlock(); + put_page(page); + + goto again; + } + + key->pointer = futex_get_inode_uuid(inode); + key->index = (unsigned long)page_to_pgoff(tail); + key->offset |= FUT_OFF_INODE; + + rcu_read_unlock(); + } + + put_page(page); + + return 0; +} + /** * futex_get_bucket - Check if the user address is valid, prepare internal * data and calculate the hash * @uaddr: futex user address * @key: data that uniquely identifies a futex + * @shared: is this a shared futex? + * + * For private futexes, each uaddr will be unique for a given mm_struct, and it + * won't be freed for the life time of the process. For shared futexes, check + * futex_get_shared_key(). * * Return: address of bucket on success, error code otherwise */ static struct futex_bucket *futex_get_bucket(void __user *uaddr, - struct futex_key *key) + struct futex_key *key, + bool shared) { uintptr_t address = (uintptr_t)uaddr; u32 hash_key; @@ -151,6 +340,9 @@ static struct futex_bucket *futex_get_bucket(void __user *uaddr, key->pointer = (u64)address; key->index = (unsigned long)current->mm; + if (shared) + futex_get_shared_key(address, current->mm, key); + /* Generate hash key for this futex using uaddr and current->mm */ hash_key = jhash2((u32 *)key, sizeof(*key) / sizeof(u32), 0); @@ -288,6 +480,7 @@ static int futex_enqueue(struct futex_waiter_head *futexv, unsigned int nr_futex int i, ret; u32 uval, val; u32 __user *uaddr; + bool retry = false; struct futex_bucket *bucket; retry: @@ -297,6 +490,18 @@ static int futex_enqueue(struct futex_waiter_head *futexv, unsigned int nr_futex uaddr = (u32 __user *)futexv->objects[i].uaddr; val = (u32)futexv->objects[i].val; + if (is_object_shared && retry) { + struct futex_bucket *tmp = + futex_get_bucket((void __user *)uaddr, + &futexv->objects[i].key, true); + if (IS_ERR(tmp)) { + __set_current_state(TASK_RUNNING); + futex_dequeue_multiple(futexv, i); + return PTR_ERR(tmp); + } + futexv->objects[i].bucket = tmp; + } + bucket = futexv->objects[i].bucket; bucket_inc_waiters(bucket); @@ -317,6 +522,7 @@ static int futex_enqueue(struct futex_waiter_head *futexv, unsigned int nr_futex if (__get_user(uval, uaddr)) return -EFAULT; + retry = true; goto retry; } @@ -430,6 +636,7 @@ static int __futex_waitv(struct futex_waiter_head *futexv, unsigned int nr_futex static long ksys_futex_wait(void __user *uaddr, unsigned int val, unsigned int flags, struct __kernel_timespec __user *timo) { + bool shared = (flags & FUTEX_SHARED_FLAG) ? true : false; unsigned int size = flags & FUTEX_SIZE_MASK; struct futex_waiter *waiter; struct futex_waiter_head *futexv; @@ -459,7 +666,7 @@ static long ksys_futex_wait(void __user *uaddr, unsigned int val, unsigned int f INIT_LIST_HEAD(&waiter->list); /* Get an unlocked hash bucket */ - waiter->bucket = futex_get_bucket(uaddr, &waiter->key); + waiter->bucket = futex_get_bucket(uaddr, &waiter->key, shared); if (IS_ERR(waiter->bucket)) return PTR_ERR(waiter->bucket); @@ -482,7 +689,6 @@ static long ksys_futex_wait(void __user *uaddr, unsigned int val, unsigned int f SYSCALL_DEFINE4(futex_wait, void __user *, uaddr, unsigned int, val, unsigned int, flags, struct __kernel_timespec __user *, timo) { - return ksys_futex_wait(uaddr, val, flags, timo); } @@ -545,6 +751,7 @@ static inline bool futex_match(struct futex_key key1, struct futex_key key2) SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, unsigned int, flags) { + bool shared = (flags & FUTEX_SHARED_FLAG) ? true : false; unsigned int size = flags & FUTEX_SIZE_MASK; struct futex_waiter waiter, *aux, *tmp; struct futex_bucket *bucket; @@ -557,7 +764,7 @@ SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, if (size != FUTEX_32) return -EINVAL; - bucket = futex_get_bucket(uaddr, &waiter.key); + bucket = futex_get_bucket(uaddr, &waiter.key, shared); if (IS_ERR(bucket)) return PTR_ERR(bucket); From d07a3bccf9236c216f402d7b31106ec0691bc204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:00 -0300 Subject: [PATCH 09/48] futex2: Implement vectorized wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support to wait on multiple futexes. This is the interface implemented by this syscall: futex_waitv(struct futex_waitv *waiters, unsigned int nr_futexes, unsigned int flags, struct timespec *timo) struct futex_waitv { __u64 val; void *uaddr; unsigned int flags; }; Given an array of struct futex_waitv, wait on each uaddr. The thread wakes if a futex_wake() is performed at any uaddr. The syscall returns immediately if any waiter has *uaddr != val. *timo is an optional timeout value for the operation. The flags argument of the syscall should be used solely for specifying the timeout as realtime, if needed. Flags for shared futexes, sizes, etc. should be used on the individual flags of each waiter. Returns the array index of one of the awakened futexes. There’s no given information of how many were awakened, or any particular attribute of it (if it’s the first awakened, if it is of the smaller index...). Signed-off-by: André Almeida --- arch/arm/tools/syscall.tbl | 1 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 2 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + include/linux/compat.h | 13 +- include/linux/syscalls.h | 4 + include/uapi/asm-generic/unistd.h | 5 +- include/uapi/linux/futex.h | 14 ++ kernel/futex2.c | 177 ++++++++++++++++++ kernel/sys_ni.c | 2 + tools/include/uapi/asm-generic/unistd.h | 5 +- .../arch/x86/entry/syscalls/syscall_64.tbl | 1 + 13 files changed, 221 insertions(+), 7 deletions(-) diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 9ac7edbb70b659..a8bb6bb54b7a45 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -456,3 +456,4 @@ 440 common process_madvise sys_process_madvise 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake +443 common futex_waitv sys_futex_waitv diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 949788f5ba4007..d1f7d35f986ea7 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 443 +#define __NR_compat_syscalls 444 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index df008993b00fda..45bc9f75ad7c53 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -893,6 +893,8 @@ __SYSCALL(__NR_process_madvise, sys_process_madvise) __SYSCALL(__NR_futex_wait, sys_futex_wait) #define __NR_futex_wake 442 __SYSCALL(__NR_futex_wake, sys_futex_wake) +#define __NR_futex_waitv 443 +__SYSCALL(__NR_futex_waitv, compat_sys_futex_waitv) /* * Please add new compat syscalls above this comment and update diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 83a75ff39b1f3b..447c4798678cdf 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -447,3 +447,4 @@ 440 i386 process_madvise sys_process_madvise 441 i386 futex_wait sys_futex_wait 442 i386 futex_wake sys_futex_wake +443 i386 futex_waitv sys_futex_waitv compat_sys_futex_waitv diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 6658fd63c7b897..f30811b56683c2 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -364,6 +364,7 @@ 440 common process_madvise sys_process_madvise 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake +443 common futex_waitv sys_futex_waitv # # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/compat.h b/include/linux/compat.h index e63005233f2763..94c68739821f3a 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -365,6 +365,12 @@ struct compat_robust_list_head { compat_uptr_t list_op_pending; }; +struct compat_futex_waitv { + compat_uptr_t uaddr; + compat_uint_t val; + compat_uint_t flags; +}; + #ifdef CONFIG_COMPAT_OLD_SIGACTION struct compat_old_sigaction { compat_uptr_t sa_handler; @@ -617,16 +623,15 @@ compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr, compat_size_t __user *len_ptr); /* kernel/futex2.c */ -asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, - compat_uint_t nr_futexes, compat_uint_t flags, - struct __kernel_timespec __user *timo); - asmlinkage long compat_sys_futex_requeue(struct compat_futex_requeue *uaddr1, struct compat_futex_requeue *uaddr2, compat_uint_t nr_wake, compat_uint_t nr_requeue, compat_u64 cmpval, compat_uint_t flags); +asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, + compat_uint_t nr_futexes, compat_uint_t flags, + struct __kernel_timespec __user *timo); /* kernel/itimer.c */ asmlinkage long compat_sys_getitimer(int which, diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index c03bc7da78c075..4ce3167814cc8a 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -68,6 +68,7 @@ union bpf_attr; struct io_uring_params; struct clone_args; struct open_how; +struct futex_waitv; #include #include @@ -618,6 +619,9 @@ asmlinkage long sys_futex_wait(void __user *uaddr, unsigned int val, unsigned in struct __kernel_timespec __user *timo); asmlinkage long sys_futex_wake(void __user *uaddr, unsigned int nr_wake, unsigned int flags); +asmlinkage long sys_futex_waitv(struct futex_waitv __user *waiters, + unsigned int nr_futexes, unsigned int flags, + struct __kernel_timespec __user *timo); /* kernel/hrtimer.c */ asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp, diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index ae47d6a9e9d244..9a631809c27372 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -866,8 +866,11 @@ __SYSCALL(__NR_futex_wait, sys_futex_wait) #define __NR_futex_wake 442 __SYSCALL(__NR_futex_wake, sys_futex_wake) +#define __NR_futex_waitv 443 +__SC_COMP(__NR_futex_waitv, sys_futex_waitv, compat_sys_futex_waitv) + #undef __NR_syscalls -#define __NR_syscalls 443 +#define __NR_syscalls 444 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 197bd5d329ebbb..a30475cc338419 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -49,6 +49,20 @@ #define FUTEX_SHARED_FLAG 8 +#define FUTEX_WAITV_MAX 128 + +/** + * struct futex_waitv - A waiter for vectorized wait + * @val: Expected value at uaddr + * @uaddr: User address to wait on + * @flags: Flags for this waiter + */ +struct futex_waitv { + void __user *uaddr; + unsigned int val; + unsigned int flags; +}; + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/kernel/futex2.c b/kernel/futex2.c index a1a31f3e41f6fe..e5e02c330567a6 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -82,6 +82,12 @@ struct futex_bucket { /* Mask for futex2 flag operations */ #define FUTEX2_MASK (FUTEX_SIZE_MASK | FUTEX_CLOCK_REALTIME | FUTEX_SHARED_FLAG) +/* Mask for sys_futex_waitv flag */ +#define FUTEXV_MASK (FUTEX_CLOCK_REALTIME) + +/* Mask for each futex in futex_waitv list */ +#define FUTEXV_WAITER_MASK (FUTEX_SIZE_MASK | FUTEX_SHARED_FLAG) + #define is_object_shared ((futexv->objects[i].flags & FUTEX_SHARED_FLAG) ? true : false) #define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ @@ -692,6 +698,177 @@ SYSCALL_DEFINE4(futex_wait, void __user *, uaddr, unsigned int, val, unsigned in return ksys_futex_wait(uaddr, val, flags, timo); } +#ifdef CONFIG_COMPAT +/** + * compat_futex_parse_waitv - Parse a waitv array from userspace + * @futexv: Kernel side list of waiters to be filled + * @uwaitv: Userspace list to be parsed + * @nr_futexes: Length of futexv + * + * Return: Error code on failure, pointer to a prepared futexv otherwise + */ +static int compat_futex_parse_waitv(struct futex_waiter_head *futexv, + struct compat_futex_waitv __user *uwaitv, + unsigned int nr_futexes) +{ + struct compat_futex_waitv waitv; + struct futex_bucket *bucket; + unsigned int i; + + for (i = 0; i < nr_futexes; i++) { + if (copy_from_user(&waitv, &uwaitv[i], sizeof(waitv))) + return -EFAULT; + + if ((waitv.flags & ~FUTEXV_WAITER_MASK) || + (waitv.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + futexv->objects[i].key.pointer = 0; + futexv->objects[i].flags = waitv.flags; + futexv->objects[i].uaddr = compat_ptr(waitv.uaddr); + futexv->objects[i].val = waitv.val; + futexv->objects[i].index = i; + + bucket = futex_get_bucket(compat_ptr(waitv.uaddr), + &futexv->objects[i].key, + is_object_shared); + + if (IS_ERR(bucket)) + return PTR_ERR(bucket); + + futexv->objects[i].bucket = bucket; + + INIT_LIST_HEAD(&futexv->objects[i].list); + } + + return 0; +} + +COMPAT_SYSCALL_DEFINE4(futex_waitv, struct compat_futex_waitv __user *, waiters, + unsigned int, nr_futexes, unsigned int, flags, + struct __kernel_timespec __user *, timo) +{ + struct futex_waiter_head *futexv; + int ret; + + if (flags & ~FUTEXV_MASK) + return -EINVAL; + + if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters) + return -EINVAL; + + futexv = kmalloc((sizeof(struct futex_waiter) * nr_futexes) + + sizeof(*futexv), GFP_KERNEL); + if (!futexv) + return -ENOMEM; + + futexv->hint = false; + futexv->task = current; + + ret = compat_futex_parse_waitv(futexv, waiters, nr_futexes); + + if (!ret) + ret = __futex_waitv(futexv, nr_futexes, timo, flags); + + kfree(futexv); + + return ret; +} +#endif + +/** + * futex_parse_waitv - Parse a waitv array from userspace + * @futexv: Kernel side list of waiters to be filled + * @uwaitv: Userspace list to be parsed + * @nr_futexes: Length of futexv + * + * Return: Error code on failure, pointer to a prepared futexv otherwise + */ +static int futex_parse_waitv(struct futex_waiter_head *futexv, + struct futex_waitv __user *uwaitv, + unsigned int nr_futexes) +{ + struct futex_bucket *bucket; + struct futex_waitv waitv; + unsigned int i; + + for (i = 0; i < nr_futexes; i++) { + if (copy_from_user(&waitv, &uwaitv[i], sizeof(waitv))) + return -EFAULT; + + if ((waitv.flags & ~FUTEXV_WAITER_MASK) || + (waitv.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + futexv->objects[i].key.pointer = 0; + futexv->objects[i].flags = waitv.flags; + futexv->objects[i].uaddr = waitv.uaddr; + futexv->objects[i].val = waitv.val; + futexv->objects[i].index = i; + + bucket = futex_get_bucket(waitv.uaddr, &futexv->objects[i].key, + is_object_shared); + + if (IS_ERR(bucket)) + return PTR_ERR(bucket); + + futexv->objects[i].bucket = bucket; + + INIT_LIST_HEAD(&futexv->objects[i].list); + } + + return 0; +} + +/** + * sys_futex_waitv - Wait on a list of futexes + * @waiters: List of futexes to wait on + * @nr_futexes: Length of futexv + * @flags: Flag for timeout (monotonic/realtime) + * @timo: Optional absolute timeout. + * + * Given an array of `struct futex_waitv`, wait on each uaddr. The thread wakes + * if a futex_wake() is performed at any uaddr. The syscall returns immediately + * if any waiter has *uaddr != val. *timo is an optional timeout value for the + * operation. Each waiter has individual flags. The `flags` argument for the + * syscall should be used solely for specifying the timeout as realtime, if + * needed. Flags for shared futexes, sizes, etc. should be used on the + * individual flags of each waiter. + * + * Returns the array index of one of the awaken futexes. There's no given + * information of how many were awakened, or any particular attribute of it (if + * it's the first awakened, if it is of the smaller index...). + */ +SYSCALL_DEFINE4(futex_waitv, struct futex_waitv __user *, waiters, + unsigned int, nr_futexes, unsigned int, flags, + struct __kernel_timespec __user *, timo) +{ + struct futex_waiter_head *futexv; + int ret; + + if (flags & ~FUTEXV_MASK) + return -EINVAL; + + if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters) + return -EINVAL; + + futexv = kmalloc((sizeof(struct futex_waiter) * nr_futexes) + + sizeof(*futexv), GFP_KERNEL); + if (!futexv) + return -ENOMEM; + + futexv->hint = false; + futexv->task = current; + + ret = futex_parse_waitv(futexv, waiters, nr_futexes); + if (!ret) + ret = __futex_waitv(futexv, nr_futexes, timo, flags); + + kfree(futexv); + + return ret; +} + /** * futex_get_parent - For a given futex in a futexv list, get a pointer to the futexv * @waiter: Address of futex in the list diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 35ff743b17d0ec..aae5642b819a0e 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -151,6 +151,8 @@ COND_SYSCALL_COMPAT(get_robust_list); /* kernel/futex2.c */ COND_SYSCALL(futex_wait); COND_SYSCALL(futex_wake); +COND_SYSCALL(futex_waitv); +COND_SYSCALL_COMPAT(futex_waitv); /* kernel/hrtimer.c */ diff --git a/tools/include/uapi/asm-generic/unistd.h b/tools/include/uapi/asm-generic/unistd.h index ae47d6a9e9d244..9a631809c27372 100644 --- a/tools/include/uapi/asm-generic/unistd.h +++ b/tools/include/uapi/asm-generic/unistd.h @@ -866,8 +866,11 @@ __SYSCALL(__NR_futex_wait, sys_futex_wait) #define __NR_futex_wake 442 __SYSCALL(__NR_futex_wake, sys_futex_wake) +#define __NR_futex_waitv 443 +__SC_COMP(__NR_futex_waitv, sys_futex_waitv, compat_sys_futex_waitv) + #undef __NR_syscalls -#define __NR_syscalls 443 +#define __NR_syscalls 444 /* * 32 bit systems traditionally used different diff --git a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl index 47de3bf93fe6f9..bd47f368f49a2d 100644 --- a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl +++ b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl @@ -364,6 +364,7 @@ 440 common process_madvise sys_process_madvise 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake +443 common futex_waitv sys_futex_waitv # # Due to a historical design error, certain syscalls are numbered differently From d2c6dd6fbd331327328124811966bd201ff2a212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:01 -0300 Subject: [PATCH 10/48] futex2: Implement requeue operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement requeue interface similarly to FUTEX_CMP_REQUEUE operation. This is the syscall implemented by this patch: futex_requeue(struct futex_requeue *uaddr1, struct futex_requeue *uaddr2, unsigned int nr_wake, unsigned int nr_requeue, u64 cmpval, unsigned int flags) struct futex_requeue { void *uaddr; unsigned int flags; }; If (uaddr1->uaddr == cmpval), wake at uaddr1->uaddr a nr_wake number of waiters and then, remove a number of nr_requeue waiters at uaddr1->uaddr and add them to uaddr2->uaddr list. Each uaddr has its own set of flags, that must be defined at struct futex_requeue (such as size, shared, NUMA). The flags argument of the syscall is there just for the sake of extensibility, and right now it needs to be zero. Return the number of the woken futexes + the number of requeued ones on success, error code otherwise. Signed-off-by: André Almeida --- The original FUTEX_CMP_REQUEUE interfaces is such as follows: futex(*uaddr1, FUTEX_CMP_REQUEUE, nr_wake, nr_requeue, *uaddr2, cmpval); Given that when this interface was created they was only one type of futex (as opposed to futex2, where there is shared, sizes, and NUMA), there was no way to specify individual flags for uaddr1 and 2. When FUTEX_PRIVATE was implemented, a new opcode was created as well (FUTEX_CMP_REQUEUE_PRIVATE), but they apply both futexes, so they should be of the same type regarding private/shared. This imposes a limitation on the use cases of the operation, and to overcome that at futex2, `struct futex_requeue` was created, so one can set individual flags for each futex. This flexibility is a trade-off with performance, given that now we need to perform two extra copy_from_user(). One alternative would be to use the upper half of flags bits to the first one, and the bottom half for the second futex, but this would also impose limitations, given that we would limit by half the flags possibilities. If equal futexes are common enough, the following extension could be added to overcome the current performance: - A flag FUTEX_REQUEUE_EQUAL is added to futex2() flags; - If futex_requeue() see this flag, that means that both futexes uses the same set of attributes. - Then, the function parses the flags as of futex_wait/wake(). - *uaddr1 and *uaddr2 are used as void* (instead of struct futex_requeue) just like wait/wake(). In that way, we could avoid the copy_from_user(). --- arch/arm/tools/syscall.tbl | 1 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 2 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + include/linux/compat.h | 12 + include/linux/syscalls.h | 5 + include/uapi/asm-generic/unistd.h | 5 +- include/uapi/linux/futex.h | 10 + kernel/futex2.c | 215 ++++++++++++++++++ kernel/sys_ni.c | 2 + .../arch/x86/entry/syscalls/syscall_64.tbl | 1 + 12 files changed, 255 insertions(+), 2 deletions(-) diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index a8bb6bb54b7a45..216d17bca682be 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -457,3 +457,4 @@ 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake 443 common futex_waitv sys_futex_waitv +444 common futex_requeue sys_futex_requeue diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index d1f7d35f986ea7..64ebdc1ec5814d 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 444 +#define __NR_compat_syscalls 445 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 45bc9f75ad7c53..2fc5ff61108376 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -895,6 +895,8 @@ __SYSCALL(__NR_futex_wait, sys_futex_wait) __SYSCALL(__NR_futex_wake, sys_futex_wake) #define __NR_futex_waitv 443 __SYSCALL(__NR_futex_waitv, compat_sys_futex_waitv) +#define __NR_futex_waitv 444 +__SYSCALL(__NR_futex_requeue, compat_sys_futex_requeue) /* * Please add new compat syscalls above this comment and update diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 447c4798678cdf..bc1b4d13d4bb7d 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -448,3 +448,4 @@ 441 i386 futex_wait sys_futex_wait 442 i386 futex_wake sys_futex_wake 443 i386 futex_waitv sys_futex_waitv compat_sys_futex_waitv +444 i386 futex_requeue sys_futex_requeue compat_sys_futex_requeue diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index f30811b56683c2..a916db190e57f5 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -365,6 +365,7 @@ 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake 443 common futex_waitv sys_futex_waitv +444 common futex_requeue sys_futex_requeue # # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/compat.h b/include/linux/compat.h index 94c68739821f3a..92932a9bf68818 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -371,6 +371,11 @@ struct compat_futex_waitv { compat_uint_t flags; }; +struct compat_futex_requeue { + compat_uptr_t uaddr; + compat_uint_t flags; +}; + #ifdef CONFIG_COMPAT_OLD_SIGACTION struct compat_old_sigaction { compat_uptr_t sa_handler; @@ -633,6 +638,13 @@ asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, compat_uint_t nr_futexes, compat_uint_t flags, struct __kernel_timespec __user *timo); +asmlinkage long compat_sys_futex_requeue(struct compat_futex_requeue *uaddr1, + struct compat_futex_requeue *uaddr2, + compat_uint_t nr_wake, + compat_uint_t nr_requeue, + compat_u64 cmpval, + compat_uint_t flags); + /* kernel/itimer.c */ asmlinkage long compat_sys_getitimer(int which, struct old_itimerval32 __user *it); diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 4ce3167814cc8a..d80eba8a43e544 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -69,6 +69,7 @@ struct io_uring_params; struct clone_args; struct open_how; struct futex_waitv; +struct futex_requeue; #include #include @@ -622,6 +623,10 @@ asmlinkage long sys_futex_wake(void __user *uaddr, unsigned int nr_wake, asmlinkage long sys_futex_waitv(struct futex_waitv __user *waiters, unsigned int nr_futexes, unsigned int flags, struct __kernel_timespec __user *timo); +asmlinkage long sys_futex_requeue(struct futex_requeue __user *uaddr1, + struct futex_requeue __user *uaddr2, + unsigned int nr_wake, unsigned int nr_requeue, + u64 cmpval, unsigned int flags); /* kernel/hrtimer.c */ asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp, diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 9a631809c27372..86e6fb7c988cb3 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -869,8 +869,11 @@ __SYSCALL(__NR_futex_wake, sys_futex_wake) #define __NR_futex_waitv 443 __SC_COMP(__NR_futex_waitv, sys_futex_waitv, compat_sys_futex_waitv) +#define __NR_futex_requeue 444 +__SC_COMP(__NR_futex_requeue, sys_futex_requeue, compat_sys_futex_requeue) + #undef __NR_syscalls -#define __NR_syscalls 444 +#define __NR_syscalls 445 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index a30475cc338419..fcc2eb8145be1e 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -63,6 +63,16 @@ struct futex_waitv { unsigned int flags; }; +/** + * struct futex_requeue - Define an address and its flags for requeue operation + * @uaddr: User address of one of the requeue arguments + * @flags: Flags for this address + */ +struct futex_requeue { + void __user *uaddr; + unsigned int flags; +}; + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/kernel/futex2.c b/kernel/futex2.c index e5e02c330567a6..3d35674b7b6b13 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -963,6 +963,221 @@ SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, return ret; } +static void futex_double_unlock(struct futex_bucket *b1, struct futex_bucket *b2) +{ + spin_unlock(&b1->lock); + if (b1 != b2) + spin_unlock(&b2->lock); +} + +static inline int __futex_requeue(struct futex_requeue rq1, + struct futex_requeue rq2, unsigned int nr_wake, + unsigned int nr_requeue, u64 cmpval, + bool shared1, bool shared2) +{ + struct futex_waiter w1, w2, *aux, *tmp; + bool retry = false; + struct futex_bucket *b1, *b2; + DEFINE_WAKE_Q(wake_q); + u32 uval; + int ret; + + b1 = futex_get_bucket(rq1.uaddr, &w1.key, shared1); + if (IS_ERR(b1)) + return PTR_ERR(b1); + + b2 = futex_get_bucket(rq2.uaddr, &w2.key, shared2); + if (IS_ERR(b2)) + return PTR_ERR(b2); + +retry: + if (shared1 && retry) { + b1 = futex_get_bucket(rq1.uaddr, &w1.key, shared1); + if (IS_ERR(b1)) + return PTR_ERR(b1); + } + + if (shared2 && retry) { + b2 = futex_get_bucket(rq2.uaddr, &w2.key, shared2); + if (IS_ERR(b2)) + return PTR_ERR(b2); + } + + bucket_inc_waiters(b2); + /* + * To ensure the locks are taken in the same order for all threads (and + * thus avoiding deadlocks), take the "smaller" one first + */ + if (b1 <= b2) { + spin_lock(&b1->lock); + if (b1 < b2) + spin_lock_nested(&b2->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock(&b2->lock); + spin_lock_nested(&b1->lock, SINGLE_DEPTH_NESTING); + } + + ret = futex_get_user(&uval, rq1.uaddr); + + if (unlikely(ret)) { + futex_double_unlock(b1, b2); + if (__get_user(uval, (u32 __user *)rq1.uaddr)) + return -EFAULT; + + bucket_dec_waiters(b2); + retry = true; + goto retry; + } + + if (uval != cmpval) { + futex_double_unlock(b1, b2); + + bucket_dec_waiters(b2); + return -EAGAIN; + } + + list_for_each_entry_safe(aux, tmp, &b1->list, list) { + if (futex_match(w1.key, aux->key)) { + if (ret < nr_wake) { + futex_mark_wake(aux, b1, &wake_q); + ret++; + continue; + } + + if (ret >= nr_wake + nr_requeue) + break; + + aux->key.pointer = w2.key.pointer; + aux->key.index = w2.key.index; + aux->key.offset = w2.key.offset; + + if (b1 != b2) { + list_del_init(&aux->list); + bucket_dec_waiters(b1); + + list_add_tail(&aux->list, &b2->list); + bucket_inc_waiters(b2); + } + ret++; + } + } + + futex_double_unlock(b1, b2); + wake_up_q(&wake_q); + bucket_dec_waiters(b2); + + return ret; +} + +#ifdef CONFIG_COMPAT +static int compat_futex_parse_requeue(struct futex_requeue *rq, + struct compat_futex_requeue __user *uaddr, + bool *shared) +{ + struct compat_futex_requeue tmp; + + if (copy_from_user(&tmp, uaddr, sizeof(tmp))) + return -EFAULT; + + if (tmp.flags & ~FUTEXV_WAITER_MASK || + (tmp.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + *shared = (tmp.flags & FUTEX_SHARED_FLAG) ? true : false; + + rq->uaddr = compat_ptr(tmp.uaddr); + rq->flags = tmp.flags; + + return 0; +} + +COMPAT_SYSCALL_DEFINE6(futex_requeue, struct compat_futex_requeue __user *, uaddr1, + struct compat_futex_requeue __user *, uaddr2, + unsigned int, nr_wake, unsigned int, nr_requeue, + compat_u64, cmpval, unsigned int, flags) +{ + struct futex_requeue rq1, rq2; + bool shared1, shared2; + int ret; + + if (flags) + return -EINVAL; + + ret = compat_futex_parse_requeue(&rq1, uaddr1, &shared1); + if (ret) + return ret; + + ret = compat_futex_parse_requeue(&rq2, uaddr2, &shared2); + if (ret) + return ret; + + return __futex_requeue(rq1, rq2, nr_wake, nr_requeue, cmpval, shared1, shared2); +} +#endif + +/** + * futex_parse_requeue - Copy a user struct futex_requeue and check it's flags + * @rq: Kernel struct + * @uaddr: Address of user struct + * @shared: Out parameter, defines if this is a shared futex + * + * Return: 0 on success, error code otherwise + */ +static int futex_parse_requeue(struct futex_requeue *rq, + struct futex_requeue __user *uaddr, bool *shared) +{ + if (copy_from_user(rq, uaddr, sizeof(*rq))) + return -EFAULT; + + if (rq->flags & ~FUTEXV_WAITER_MASK || + (rq->flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + *shared = (rq->flags & FUTEX_SHARED_FLAG) ? true : false; + + return 0; +} + +/** + * sys_futex_requeue - Wake futexes at uaddr1 and requeue from uaddr1 to uaddr2 + * @uaddr1: Address of futexes to be waken/dequeued + * @uaddr2: Address for the futexes to be enqueued + * @nr_wake: Number of futexes waiting in uaddr1 to be woken up + * @nr_requeue: Number of futexes to be requeued from uaddr1 to uaddr2 + * @cmpval: Expected value at uaddr1 + * @flags: Reserved flags arg for requeue operation expansion. Must be 0. + * + * If (uaddr1->uaddr == cmpval), wake at uaddr1->uaddr a nr_wake number of + * waiters and then, remove a number of nr_requeue waiters at uaddr1->uaddr + * and add then to uaddr2->uaddr list. Each uaddr has its own set of flags, + * that must be defined at struct futex_requeue (such as size, shared, NUMA). + * + * Return the number of the woken futexes + the number of requeued ones on + * success, error code otherwise. + */ +SYSCALL_DEFINE6(futex_requeue, struct futex_requeue __user *, uaddr1, + struct futex_requeue __user *, uaddr2, + unsigned int, nr_wake, unsigned int, nr_requeue, + u64, cmpval, unsigned int, flags) +{ + struct futex_requeue rq1, rq2; + bool shared1, shared2; + int ret; + + if (flags) + return -EINVAL; + + ret = futex_parse_requeue(&rq1, uaddr1, &shared1); + if (ret) + return ret; + + ret = futex_parse_requeue(&rq2, uaddr2, &shared2); + if (ret) + return ret; + + return __futex_requeue(rq1, rq2, nr_wake, nr_requeue, cmpval, shared1, shared2); +} + static int __init futex2_init(void) { int i; diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index aae5642b819a0e..1746d5dda0b8e2 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -153,6 +153,8 @@ COND_SYSCALL(futex_wait); COND_SYSCALL(futex_wake); COND_SYSCALL(futex_waitv); COND_SYSCALL_COMPAT(futex_waitv); +COND_SYSCALL(futex_requeue); +COND_SYSCALL_COMPAT(futex_requeue); /* kernel/hrtimer.c */ diff --git a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl index bd47f368f49a2d..e3083cd67b8d20 100644 --- a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl +++ b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl @@ -365,6 +365,7 @@ 441 common futex_wait sys_futex_wait 442 common futex_wake sys_futex_wake 443 common futex_waitv sys_futex_waitv +444 common futex_requeue sys_futex_requeue # # Due to a historical design error, certain syscalls are numbered differently From 0a3d1d9585b593f01ef3657584957446791011e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Thu, 11 Feb 2021 10:47:23 -0300 Subject: [PATCH 11/48] futex2: Add compatibility entry point for x86_x32 ABI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New syscalls should use the same entry point for x86_64 and x86_x32 paths. Add a wrapper for x32 calls to use parse functions that assumes 32bit pointers. Signed-off-by: André Almeida --- kernel/futex2.c | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/kernel/futex2.c b/kernel/futex2.c index 3d35674b7b6b13..49f474bddb76cf 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -23,6 +23,10 @@ #include #include +#ifdef CONFIG_X86_64 +#include +#endif + /** * struct futex_key - Components to build unique key for a futex * @pointer: Pointer to current->mm or inode's UUID for file backed futexes @@ -860,7 +864,16 @@ SYSCALL_DEFINE4(futex_waitv, struct futex_waitv __user *, waiters, futexv->hint = false; futexv->task = current; - ret = futex_parse_waitv(futexv, waiters, nr_futexes); +#ifdef CONFIG_X86_X32_ABI + if (in_x32_syscall()) { + ret = compat_futex_parse_waitv(futexv, (struct compat_futex_waitv *)waiters, + nr_futexes); + } else +#endif + { + ret = futex_parse_waitv(futexv, waiters, nr_futexes); + } + if (!ret) ret = __futex_waitv(futexv, nr_futexes, timo, flags); @@ -1167,13 +1180,28 @@ SYSCALL_DEFINE6(futex_requeue, struct futex_requeue __user *, uaddr1, if (flags) return -EINVAL; - ret = futex_parse_requeue(&rq1, uaddr1, &shared1); - if (ret) - return ret; +#ifdef CONFIG_X86_X32_ABI + if (in_x32_syscall()) { + ret = compat_futex_parse_requeue(&rq1, (struct compat_futex_requeue *)uaddr1, + &shared1); + if (ret) + return ret; - ret = futex_parse_requeue(&rq2, uaddr2, &shared2); - if (ret) - return ret; + ret = compat_futex_parse_requeue(&rq2, (struct compat_futex_requeue *)uaddr2, + &shared2); + if (ret) + return ret; + } else +#endif + { + ret = futex_parse_requeue(&rq1, uaddr1, &shared1); + if (ret) + return ret; + + ret = futex_parse_requeue(&rq2, uaddr2, &shared2); + if (ret) + return ret; + } return __futex_requeue(rq1, rq2, nr_wake, nr_requeue, cmpval, shared1, shared2); } From b3a2926623fb827c2c17cada985b0f2a777b16ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Tue, 9 Feb 2021 13:59:00 -0300 Subject: [PATCH 12/48] docs: locking: futex2: Add documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new documentation file specifying both userspace API and internal implementation details of futex2 syscalls. Signed-off-by: André Almeida --- Documentation/locking/futex2.rst | 198 +++++++++++++++++++++++++++++++ Documentation/locking/index.rst | 1 + 2 files changed, 199 insertions(+) create mode 100644 Documentation/locking/futex2.rst diff --git a/Documentation/locking/futex2.rst b/Documentation/locking/futex2.rst new file mode 100644 index 00000000000000..2f74d7c97a5579 --- /dev/null +++ b/Documentation/locking/futex2.rst @@ -0,0 +1,198 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====== +futex2 +====== + +:Author: André Almeida + +futex, or fast user mutex, is a set of syscalls to allow userspace to create +performant synchronization mechanisms, such as mutexes, semaphores and +conditional variables in userspace. C standard libraries, like glibc, uses it +as a means to implement more high level interfaces like pthreads. + +The interface +============= + +uAPI functions +-------------- + +.. kernel-doc:: kernel/futex2.c + :identifiers: sys_futex_wait sys_futex_wake sys_futex_waitv sys_futex_requeue + +uAPI structures +--------------- + +.. kernel-doc:: include/uapi/linux/futex.h + +The ``flag`` argument +--------------------- + +The flag is used to specify the size of the futex word +(FUTEX_[8, 16, 32, 64]). It's mandatory to define one, since there's no +default size. + +By default, the timeout uses a monotonic clock, but can be used as a realtime +one by using the FUTEX_REALTIME_CLOCK flag. + +By default, futexes are of the private type, that means that this user address +will be accessed by threads that share the same memory region. This allows for +some internal optimizations, so they are faster. However, if the address needs +to be shared with different processes (like using ``mmap()`` or ``shm()``), they +need to be defined as shared and the flag FUTEX_SHARED_FLAG is used to set that. + +By default, the operation has no NUMA-awareness, meaning that the user can't +choose the memory node where the kernel side futex data will be stored. The +user can choose the node where it wants to operate by setting the +FUTEX_NUMA_FLAG and using the following structure (where X can be 8, 16, 32 or +64):: + + struct futexX_numa { + __uX value; + __sX hint; + }; + +This structure should be passed at the ``void *uaddr`` of futex functions. The +address of the structure will be used to be waited on/waken on, and the +``value`` will be compared to ``val`` as usual. The ``hint`` member is used to +define which node the futex will use. When waiting, the futex will be +registered on a kernel-side table stored on that node; when waking, the futex +will be searched for on that given table. That means that there's no redundancy +between tables, and the wrong ``hint`` value will lead to undesired behavior. +Userspace is responsible for dealing with node migrations issues that may +occur. ``hint`` can range from [0, MAX_NUMA_NODES), for specifying a node, or +-1, to use the same node the current process is using. + +When not using FUTEX_NUMA_FLAG on a NUMA system, the futex will be stored on a +global table on allocated on the first node. + +The ``timo`` argument +--------------------- + +As per the Y2038 work done in the kernel, new interfaces shouldn't add timeout +options known to be buggy. Given that, ``timo`` should be a 64-bit timeout at +all platforms, using an absolute timeout value. + +Implementation +============== + +The internal implementation follows a similar design to the original futex. +Given that we want to replicate the same external behavior of current futex, +this should be somewhat expected. + +Waiting +------- + +For the wait operations, they are all treated as if you want to wait on N +futexes, so the path for futex_wait and futex_waitv is the basically the same. +For both syscalls, the first step is to prepare an internal list for the list +of futexes to wait for (using struct futexv_head). For futex_wait() calls, this +list will have a single object. + +We have a hash table, where waiters register themselves before sleeping. Then +the wake function checks this table looking for waiters at uaddr. The hash +bucket to be used is determined by a struct futex_key, that stores information +to uniquely identify an address from a given process. Given the huge address +space, there'll be hash collisions, so we store information to be later used on +collision treatment. + +First, for every futex we want to wait on, we check if (``*uaddr == val``). +This check is done holding the bucket lock, so we are correctly serialized with +any futex_wake() calls. If any waiter fails the check above, we dequeue all +futexes. The check (``*uaddr == val``) can fail for two reasons: + +- The values are different, and we return -EAGAIN. However, if while + dequeueing we found that some futexes were awakened, we prioritize this + and return success. + +- When trying to access the user address, we do so with page faults + disabled because we are holding a bucket's spin lock (and can't sleep + while holding a spin lock). If there's an error, it might be a page + fault, or an invalid address. We release the lock, dequeue everyone + (because it's illegal to sleep while there are futexes enqueued, we + could lose wakeups) and try again with page fault enabled. If we + succeed, this means that the address is valid, but we need to do + all the work again. For serialization reasons, we need to have the + spin lock when getting the user value. Additionally, for shared + futexes, we also need to recalculate the hash, since the underlying + mapping mechanisms could have changed when dealing with page fault. + If, even with page fault enabled, we can't access the address, it + means it's an invalid user address, and we return -EFAULT. For this + case, we prioritize the error, even if some futexes were awaken. + +If the check is OK, they are enqueued on a linked list in our bucket, and +proceed to the next one. If all waiters succeed, we put the thread to sleep +until a futex_wake() call, timeout expires or we get a signal. After waking up, +we dequeue everyone, and check if some futex was awakened. This dequeue is done +by iteratively walking at each element of struct futex_head list. + +All enqueuing/dequeuing operations requires to hold the bucket lock, to avoid +racing while modifying the list. + +Waking +------ + +We get the bucket that's storing the waiters at uaddr, and wake the required +number of waiters, checking for hash collision. + +There's an optimization that makes futex_wake() not take the bucket lock if +there's no one to be woken on that bucket. It checks an atomic counter that each +bucket has, if it says 0, then the syscall exits. In order for this to work, the +waiter thread increases it before taking the lock, so the wake thread will +correctly see that there's someone waiting and will continue the path to take +the bucket lock. To get the correct serialization, the waiter issues a memory +barrier after increasing the bucket counter and the waker issues a memory +barrier before checking it. + +Requeuing +--------- + +The requeue path first checks for each struct futex_requeue and their flags. +Then, it will compare the expected value with the one at uaddr1::uaddr. +Following the same serialization explained at Waking_, we increase the atomic +counter for the bucket of uaddr2 before taking the lock. We need to have both +buckets locks at same time so we don't race with other futex operation. To +ensure the locks are taken in the same order for all threads (and thus avoiding +deadlocks), every requeue operation takes the "smaller" bucket first, when +comparing both addresses. + +If the compare with user value succeeds, we proceed by waking ``nr_wake`` +futexes, and then requeuing ``nr_requeue`` from bucket of uaddr1 to the uaddr2. +This consists in a simple list deletion/addition and replacing the old futex key +with the new one. + +Futex keys +---------- + +There are two types of futexes: private and shared ones. The private are futexes +meant to be used by threads that share the same memory space, are easier to be +uniquely identified and thus can have some performance optimization. The +elements for identifying one are: the start address of the page where the +address is, the address offset within the page and the current->mm pointer. + +Now, for uniquely identifying a shared futex: + +- If the page containing the user address is an anonymous page, we can + just use the same data used for private futexes (the start address of + the page, the address offset within the page and the current->mm + pointer); that will be enough for uniquely identifying such futex. We + also set one bit at the key to differentiate if a private futex is + used on the same address (mixing shared and private calls does not + work). + +- If the page is file-backed, current->mm maybe isn't the same one for + every user of this futex, so we need to use other data: the + page->index, a UUID for the struct inode and the offset within the + page. + +Note that members of futex_key don't have any particular meaning after they +are part of the struct - they are just bytes to identify a futex. Given that, +we don't need to use a particular name or type that matches the original data, +we only need to care about the bitsize of each component and make both private +and shared fit in the same memory space. + +Source code documentation +========================= + +.. kernel-doc:: kernel/futex2.c + :no-identifiers: sys_futex_wait sys_futex_wake sys_futex_waitv sys_futex_requeue diff --git a/Documentation/locking/index.rst b/Documentation/locking/index.rst index 7003bd5aeff4cf..9bf03c7fa1ec74 100644 --- a/Documentation/locking/index.rst +++ b/Documentation/locking/index.rst @@ -24,6 +24,7 @@ locking percpu-rw-semaphore robust-futexes robust-futex-ABI + futex2 .. only:: subproject and html From 8d130dd2dc8f89bfb353aa6cbe8ebbf82dabf16e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:01 -0300 Subject: [PATCH 13/48] selftests: futex2: Add wake/wait test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a simple file to test wake/wait mechanism using futex2 interface. Test three scenarios: using a common local int variable as private futex, a shm futex as shared futex and a file-backed shared memory as a shared futex. This should test all branches of futex_get_key(). Create helper files so more tests can evaluate futex2. While 32bit ABIs from glibc aren't yet able to use 64 bit sized time variables, add a temporary workaround that implements the required types and calls the appropriated syscalls, since futex2 doesn't supports 32 bit sized time. Signed-off-by: André Almeida --- .../selftests/futex/functional/.gitignore | 1 + .../selftests/futex/functional/Makefile | 4 +- .../selftests/futex/functional/futex2_wait.c | 195 ++++++++++++++++++ .../testing/selftests/futex/functional/run.sh | 3 + .../selftests/futex/include/futex2test.h | 79 +++++++ 5 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/futex/functional/futex2_wait.c create mode 100644 tools/testing/selftests/futex/include/futex2test.h diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore index 0efcd494daabfa..d61f1df943601a 100644 --- a/tools/testing/selftests/futex/functional/.gitignore +++ b/tools/testing/selftests/futex/functional/.gitignore @@ -6,3 +6,4 @@ futex_wait_private_mapped_file futex_wait_timeout futex_wait_uninitialized_heap futex_wait_wouldblock +futex2_wait diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index 23207829ec752b..7142a94a7ac318 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -5,6 +5,7 @@ LDLIBS := -lpthread -lrt HEADERS := \ ../include/futextest.h \ + ../include/futex2test.h \ ../include/atomic.h \ ../include/logging.h TEST_GEN_FILES := \ @@ -14,7 +15,8 @@ TEST_GEN_FILES := \ futex_requeue_pi_signal_restart \ futex_requeue_pi_mismatched_ops \ futex_wait_uninitialized_heap \ - futex_wait_private_mapped_file + futex_wait_private_mapped_file \ + futex2_wait TEST_PROGS := run.sh diff --git a/tools/testing/selftests/futex/functional/futex2_wait.c b/tools/testing/selftests/futex/functional/futex2_wait.c new file mode 100644 index 00000000000000..7f5516369ade41 --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex2_wait.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/****************************************************************************** + * + * Copyright Collabora Ltd., 2021 + * + * DESCRIPTION + * Test wait/wake mechanism of futex2, using 32bit sized futexes. + * + * AUTHOR + * André Almeida + * + * HISTORY + * 2021-Feb-5: Initial version by André + * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "futex2test.h" +#include "logging.h" + +#define TEST_NAME "futex2-wait" +#define timeout_ns 30000000 +#define WAKE_WAIT_US 10000 +#define SHM_PATH "futex2_shm_file" + +void *futex; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +static void *waiterfn(void *arg) +{ + struct timespec64 to64; + unsigned int flags = 0; + + if (arg) + flags = *((unsigned int *) arg); + + /* setting absolute timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_nsec += timeout_ns; + + if (to64.tv_nsec >= 1000000000) { + to64.tv_sec++; + to64.tv_nsec -= 1000000000; + } + + if (futex2_wait(futex, 0, FUTEX_32 | flags, &to64)) + printf("waiter failed errno %d\n", errno); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + unsigned int flags = FUTEX_SHARED_FLAG; + int res, ret = RET_PASS, fd, c, shm_id; + u_int32_t f_private = 0, *shared_data; + pthread_t waiter; + void *shm; + + futex = &f_private; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(3); + ksft_print_msg("%s: Test FUTEX2_WAIT\n", basename(argv[0])); + + /* Testing a private futex */ + info("Calling private futex2_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling private futex2_wake on futex: %p\n", futex); + res = futex2_wake(futex, 1, FUTEX_32); + if (res != 1) { + ksft_test_result_fail("futex2_wake private returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wake private succeeds\n"); + } + + /* Testing an anon page shared memory */ + shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666); + if (shm_id < 0) { + perror("shmget"); + exit(1); + } + + shared_data = shmat(shm_id, NULL, 0); + + *shared_data = 0; + futex = shared_data; + + info("Calling (page anon) shared futex2_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, &flags)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling (page anon) shared futex2_wake on futex: %p\n", futex); + res = futex2_wake(futex, 1, FUTEX_32 | FUTEX_SHARED_FLAG); + if (res != 1) { + ksft_test_result_fail("futex2_wake shared (page anon) returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wake shared (page anon) succeeds\n"); + } + + + /* Testing a file backed shared memory */ + fd = open(SHM_PATH, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + if (fd < 0) { + perror("open"); + exit(1); + } + + if (ftruncate(fd, sizeof(f_private))) { + perror("ftruncate"); + exit(1); + } + + shm = mmap(NULL, sizeof(f_private), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (shm == MAP_FAILED) { + perror("mmap"); + exit(1); + } + + memcpy(shm, &f_private, sizeof(f_private)); + + futex = shm; + + info("Calling shared (file backed) futex2_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, &flags)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling shared (file backed) futex2_wake on futex: %p\n", futex); + res = futex2_wake(shm, 1, FUTEX_32 | FUTEX_SHARED_FLAG); + if (res != 1) { + ksft_test_result_fail("futex2_wake shared (file backed) returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wake shared (file backed) succeeds\n"); + } + + /* Freeing resources */ + shmdt(shared_data); + munmap(shm, sizeof(f_private)); + remove(SHM_PATH); + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/functional/run.sh b/tools/testing/selftests/futex/functional/run.sh index 1acb6ace1680e8..3730159c865a1f 100755 --- a/tools/testing/selftests/futex/functional/run.sh +++ b/tools/testing/selftests/futex/functional/run.sh @@ -73,3 +73,6 @@ echo echo ./futex_wait_uninitialized_heap $COLOR ./futex_wait_private_mapped_file $COLOR + +echo +./futex2_wait $COLOR diff --git a/tools/testing/selftests/futex/include/futex2test.h b/tools/testing/selftests/futex/include/futex2test.h new file mode 100644 index 00000000000000..e724d56b917e4c --- /dev/null +++ b/tools/testing/selftests/futex/include/futex2test.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/****************************************************************************** + * + * Copyright Collabora Ltd., 2021 + * + * DESCRIPTION + * Futex2 library addons for old futex library + * + * AUTHOR + * André Almeida + * + * HISTORY + * 2021-Feb-5: Initial version by André + * + *****************************************************************************/ +#include "futextest.h" +#include + +#define NSEC_PER_SEC 1000000000L + +#ifndef FUTEX_8 +# define FUTEX_8 0 +#endif +#ifndef FUTEX_16 +# define FUTEX_16 1 +#endif +#ifndef FUTEX_32 +# define FUTEX_32 2 +#endif + +/* + * - Y2038 section for 32-bit applications - + * + * Remove this when glibc is ready for y2038. Then, always compile with + * `-DTIME_BITS=64` or `-D__USE_TIME_BITS64`. glibc will provide both + * timespec64 and clock_gettime64 so we won't need to define here. + */ +#if defined(__i386__) || __TIMESIZE == 32 +# define NR_gettime __NR_clock_gettime64 +#else +# define NR_gettime __NR_clock_gettime +#endif + +struct timespec64 { + long long tv_sec; /* seconds */ + long long tv_nsec; /* nanoseconds */ +}; + +int gettime64(clock_t clockid, struct timespec64 *tv) +{ + return syscall(NR_gettime, clockid, tv); +} +/* + * - End of Y2038 section - + */ + +/** + * futex2_wait - If (*uaddr == val), wait at uaddr until timo + * @uaddr: User address to wait on + * @val: Expected value at uaddr, return if is not equal + * @flags: Operation flags + * @timo: Optional timeout for operation + */ +static inline int futex2_wait(volatile void *uaddr, unsigned long val, + unsigned long flags, struct timespec64 *timo) +{ + return syscall(__NR_futex_wait, uaddr, val, flags, timo); +} + +/** + * futex2_wake - Wake a number of waiters at uaddr + * @uaddr: Address to wake + * @nr: Number of waiters to wake + * @flags: Operation flags + */ +static inline int futex2_wake(volatile void *uaddr, unsigned int nr, unsigned long flags) +{ + return syscall(__NR_futex_wake, uaddr, nr, flags); +} From bfe3d0df5652126bec57bab25de1c813386181b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:01 -0300 Subject: [PATCH 14/48] selftests: futex2: Add timeout test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapt existing futex wait timeout file to test the same mechanism for futex2. futex2 accepts only absolute 64bit timers, but supports both monotonic and realtime clocks. Signed-off-by: André Almeida --- .../futex/functional/futex_wait_timeout.c | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c index ee55e6d389a3f0..b4dffe9e3b449f 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c +++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c @@ -11,6 +11,7 @@ * * HISTORY * 2009-Nov-6: Initial version by Darren Hart + * 2021-Feb-5: Add futex2 test by André * *****************************************************************************/ @@ -20,7 +21,7 @@ #include #include #include -#include "futextest.h" +#include "futex2test.h" #include "logging.h" #define TEST_NAME "futex-wait-timeout" @@ -40,7 +41,8 @@ void usage(char *prog) int main(int argc, char *argv[]) { futex_t f1 = FUTEX_INITIALIZER; - struct timespec to; + struct timespec to = {.tv_sec = 0, .tv_nsec = timeout_ns}; + struct timespec64 to64; int res, ret = RET_PASS; int c; @@ -65,22 +67,60 @@ int main(int argc, char *argv[]) } ksft_print_header(); - ksft_set_plan(1); + ksft_set_plan(3); ksft_print_msg("%s: Block on a futex and wait for timeout\n", basename(argv[0])); ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns); - /* initialize timeout */ - to.tv_sec = 0; - to.tv_nsec = timeout_ns; - info("Calling futex_wait on f1: %u @ %p\n", f1, &f1); res = futex_wait(&f1, f1, &to, FUTEX_PRIVATE_FLAG); if (!res || errno != ETIMEDOUT) { - fail("futex_wait returned %d\n", ret < 0 ? errno : ret); + ksft_test_result_fail("futex_wait returned %d\n", ret < 0 ? errno : ret); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_wait timeout succeeds\n"); + } + + /* setting absolute monotonic timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_nsec += timeout_ns; + + if (to64.tv_nsec >= 1000000000) { + to64.tv_sec++; + to64.tv_nsec -= 1000000000; + } + + info("Calling futex2_wait on f1: %u @ %p\n", f1, &f1); + res = futex2_wait(&f1, f1, FUTEX_32, &to64); + if (!res || errno != ETIMEDOUT) { + ksft_test_result_fail("futex2_wait monotonic returned %d\n", ret < 0 ? errno : ret); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wait monotonic timeout succeeds\n"); + } + + /* setting absolute realtime timeout for futex2 */ + if (gettime64(CLOCK_REALTIME, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_nsec += timeout_ns; + + if (to64.tv_nsec >= 1000000000) { + to64.tv_sec++; + to64.tv_nsec -= 1000000000; + } + + info("Calling futex2_wait on f1: %u @ %p\n", f1, &f1); + res = futex2_wait(&f1, f1, FUTEX_32 | FUTEX_CLOCK_REALTIME, &to64); + if (!res || errno != ETIMEDOUT) { + ksft_test_result_fail("futex2_wait realtime returned %d\n", ret < 0 ? errno : ret); ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wait realtime timeout succeeds\n"); } - print_result(TEST_NAME, ret); + ksft_print_cnts(); return ret; } From 0d7a890b65fe41403bd2ba59680727b59367015e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:01 -0300 Subject: [PATCH 15/48] selftests: futex2: Add wouldblock test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapt existing futex wait wouldblock file to test the same mechanism for futex2. Signed-off-by: André Almeida --- .../futex/functional/futex_wait_wouldblock.c | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c index 0ae390ff816449..ed36600909072d 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c +++ b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c @@ -12,6 +12,7 @@ * * HISTORY * 2009-Nov-14: Initial version by Gowrishankar + * 2021-Feb-5: Add futex2 test by André * *****************************************************************************/ @@ -21,7 +22,7 @@ #include #include #include -#include "futextest.h" +#include "futex2test.h" #include "logging.h" #define TEST_NAME "futex-wait-wouldblock" @@ -39,6 +40,7 @@ void usage(char *prog) int main(int argc, char *argv[]) { struct timespec to = {.tv_sec = 0, .tv_nsec = timeout_ns}; + struct timespec64 to64; futex_t f1 = FUTEX_INITIALIZER; int res, ret = RET_PASS; int c; @@ -61,18 +63,41 @@ int main(int argc, char *argv[]) } ksft_print_header(); - ksft_set_plan(1); + ksft_set_plan(2); ksft_print_msg("%s: Test the unexpected futex value in FUTEX_WAIT\n", basename(argv[0])); info("Calling futex_wait on f1: %u @ %p with val=%u\n", f1, &f1, f1+1); res = futex_wait(&f1, f1+1, &to, FUTEX_PRIVATE_FLAG); if (!res || errno != EWOULDBLOCK) { - fail("futex_wait returned: %d %s\n", + ksft_test_result_fail("futex_wait returned: %d %s\n", res ? errno : res, res ? strerror(errno) : ""); ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_wait wouldblock succeeds\n"); } - print_result(TEST_NAME, ret); + /* setting absolute timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_nsec += timeout_ns; + + if (to64.tv_nsec >= 1000000000) { + to64.tv_sec++; + to64.tv_nsec -= 1000000000; + } + + info("Calling futex2_wait on f1: %u @ %p with val=%u\n", f1, &f1, f1+1); + res = futex2_wait(&f1, f1+1, FUTEX_32, &to64); + if (!res || errno != EWOULDBLOCK) { + ksft_test_result_fail("futex2_wait returned: %d %s\n", + res ? errno : res, res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wait wouldblock succeeds\n"); + } + + ksft_print_cnts(); return ret; } From 23f6a53cffe7d5a3bcd36838c6a37e6fe2a8dd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:02 -0300 Subject: [PATCH 16/48] selftests: futex2: Add waitv test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a new file to test the waitv mechanism. Test both private and shared futexes. Wake the last futex in the array, and check if the return value from futex_waitv() is the right index. Signed-off-by: André Almeida --- .../selftests/futex/functional/.gitignore | 1 + .../selftests/futex/functional/Makefile | 3 +- .../selftests/futex/functional/futex2_waitv.c | 154 ++++++++++++++++++ .../testing/selftests/futex/functional/run.sh | 3 + .../selftests/futex/include/futex2test.h | 17 ++ 5 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/futex/functional/futex2_waitv.c diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore index d61f1df943601a..d0b8f637b78664 100644 --- a/tools/testing/selftests/futex/functional/.gitignore +++ b/tools/testing/selftests/futex/functional/.gitignore @@ -7,3 +7,4 @@ futex_wait_timeout futex_wait_uninitialized_heap futex_wait_wouldblock futex2_wait +futex2_waitv diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index 7142a94a7ac318..b857b9450507ba 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -16,7 +16,8 @@ TEST_GEN_FILES := \ futex_requeue_pi_mismatched_ops \ futex_wait_uninitialized_heap \ futex_wait_private_mapped_file \ - futex2_wait + futex2_wait \ + futex2_waitv TEST_PROGS := run.sh diff --git a/tools/testing/selftests/futex/functional/futex2_waitv.c b/tools/testing/selftests/futex/functional/futex2_waitv.c new file mode 100644 index 00000000000000..412122012aefa2 --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex2_waitv.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/****************************************************************************** + * + * Copyright Collabora Ltd., 2021 + * + * DESCRIPTION + * Test waitv/wake mechanism of futex2, using 32bit sized futexes. + * + * AUTHOR + * André Almeida + * + * HISTORY + * 2021-Feb-5: Initial version by André + * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "futex2test.h" +#include "logging.h" + +#define TEST_NAME "futex2-wait" +#define WAKE_WAIT_US 10000 +#define NR_FUTEXES 30 +struct futex_waitv waitv[NR_FUTEXES]; +u_int32_t futexes[NR_FUTEXES] = {0}; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +void *waiterfn(void *arg) +{ + struct timespec64 to64; + int res; + + /* setting absolute timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_sec++; + + res = futex2_waitv(waitv, NR_FUTEXES, 0, &to64); + if (res < 0) { + ksft_test_result_fail("futex2_waitv returned: %d %s\n", + errno, strerror(errno)); + } else if (res != NR_FUTEXES - 1) { + ksft_test_result_fail("futex2_waitv returned: %d, expecting %d\n", + res, NR_FUTEXES - 1); + } + + return NULL; +} + +int main(int argc, char *argv[]) +{ + pthread_t waiter; + int res, ret = RET_PASS; + int c, i; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(2); + ksft_print_msg("%s: Test FUTEX2_WAITV\n", + basename(argv[0])); + + for (i = 0; i < NR_FUTEXES; i++) { + waitv[i].uaddr = &futexes[i]; + waitv[i].flags = FUTEX_32; + waitv[i].val = 0; + } + + /* Private waitv */ + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + res = futex2_wake(waitv[NR_FUTEXES - 1].uaddr, 1, FUTEX_32); + if (res != 1) { + ksft_test_result_fail("futex2_wake private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_wake private succeeds\n"); + } + + /* Shared waitv */ + for (i = 0; i < NR_FUTEXES; i++) { + int shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666); + + if (shm_id < 0) { + perror("shmget"); + exit(1); + } + + unsigned int *shared_data = shmat(shm_id, NULL, 0); + + *shared_data = 0; + waitv[i].uaddr = shared_data; + waitv[i].flags = FUTEX_32 | FUTEX_SHARED_FLAG; + waitv[i].val = 0; + } + + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + res = futex2_wake(waitv[NR_FUTEXES - 1].uaddr, 1, FUTEX_32 | FUTEX_SHARED_FLAG); + if (res != 1) { + ksft_test_result_fail("futex2_waitv shared returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_waitv shared succeeds\n"); + } + + for (i = 0; i < NR_FUTEXES; i++) + shmdt(waitv[i].uaddr); + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/functional/run.sh b/tools/testing/selftests/futex/functional/run.sh index 3730159c865a1f..18b3883d723646 100755 --- a/tools/testing/selftests/futex/functional/run.sh +++ b/tools/testing/selftests/futex/functional/run.sh @@ -76,3 +76,6 @@ echo echo ./futex2_wait $COLOR + +echo +./futex2_waitv $COLOR diff --git a/tools/testing/selftests/futex/include/futex2test.h b/tools/testing/selftests/futex/include/futex2test.h index e724d56b917e4c..0ed3b20935be96 100644 --- a/tools/testing/selftests/futex/include/futex2test.h +++ b/tools/testing/selftests/futex/include/futex2test.h @@ -28,6 +28,10 @@ # define FUTEX_32 2 #endif +#ifndef FUTEX_SHARED_FLAG +#define FUTEX_SHARED_FLAG 8 +#endif + /* * - Y2038 section for 32-bit applications - * @@ -77,3 +81,16 @@ static inline int futex2_wake(volatile void *uaddr, unsigned int nr, unsigned lo { return syscall(__NR_futex_wake, uaddr, nr, flags); } + +/** + * futex2_waitv - Wait at multiple futexes, wake on any + * @waiters: Array of waiters + * @nr_waiters: Length of waiters array + * @flags: Operation flags + * @timo: Optional timeout for operation + */ +static inline int futex2_waitv(volatile struct futex_waitv *waiters, unsigned long nr_waiters, + unsigned long flags, struct timespec64 *timo) +{ + return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo); +} From 10c4a56c5b2f60779d022b606d0dfcd6b5a924ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:02 -0300 Subject: [PATCH 17/48] selftests: futex2: Add requeue test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add testing for futex_requeue(). The first test just requeue from one waiter to another one, and wake it. The second performs both wake and requeue, and we check return values to see if the operation woke/requeued the expected number of waiters. Signed-off-by: André Almeida --- .../selftests/futex/functional/.gitignore | 1 + .../selftests/futex/functional/Makefile | 3 +- .../futex/functional/futex2_requeue.c | 164 ++++++++++++++++++ .../selftests/futex/include/futex2test.h | 16 ++ 4 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/futex/functional/futex2_requeue.c diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore index d0b8f637b78664..af7557e821daa7 100644 --- a/tools/testing/selftests/futex/functional/.gitignore +++ b/tools/testing/selftests/futex/functional/.gitignore @@ -8,3 +8,4 @@ futex_wait_uninitialized_heap futex_wait_wouldblock futex2_wait futex2_waitv +futex2_requeue diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index b857b9450507ba..ec0e713f0e425f 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -17,7 +17,8 @@ TEST_GEN_FILES := \ futex_wait_uninitialized_heap \ futex_wait_private_mapped_file \ futex2_wait \ - futex2_waitv + futex2_waitv \ + futex2_requeue TEST_PROGS := run.sh diff --git a/tools/testing/selftests/futex/functional/futex2_requeue.c b/tools/testing/selftests/futex/functional/futex2_requeue.c new file mode 100644 index 00000000000000..1bc3704dc8c28b --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex2_requeue.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/****************************************************************************** + * + * Copyright Collabora Ltd., 2021 + * + * DESCRIPTION + * Test requeue mechanism of futex2, using 32bit sized futexes. + * + * AUTHOR + * André Almeida + * + * HISTORY + * 2021-Feb-5: Initial version by André + * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "futex2test.h" +#include "logging.h" + +#define TEST_NAME "futex2-wait" +#define timeout_ns 30000000 +#define WAKE_WAIT_US 10000 +volatile futex_t *f1; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +void *waiterfn(void *arg) +{ + struct timespec64 to64; + + /* setting absolute timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_nsec += timeout_ns; + + if (to64.tv_nsec >= 1000000000) { + to64.tv_sec++; + to64.tv_nsec -= 1000000000; + } + + if (futex2_wait(f1, *f1, FUTEX_32, &to64)) + printf("waiter failed errno %d\n", errno); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + pthread_t waiter[10]; + int res, ret = RET_PASS; + int c, i; + volatile futex_t _f1 = 0; + volatile futex_t f2 = 0; + struct futex_requeue r1, r2; + + f1 = &_f1; + + r1.flags = FUTEX_32; + r2.flags = FUTEX_32; + + r1.uaddr = f1; + r2.uaddr = &f2; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(2); + ksft_print_msg("%s: Test FUTEX2_REQUEUE\n", + basename(argv[0])); + + /* + * Requeue a waiter from f1 to f2, and wake f2. + */ + if (pthread_create(&waiter[0], NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + res = futex2_requeue(&r1, &r2, 0, 1, 0, 0); + if (res != 1) { + ksft_test_result_fail("futex2_requeue private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } + + + info("Calling private futex2_wake on f2: %u @ %p with val=%u\n", f2, &f2, f2); + res = futex2_wake(&f2, 1, FUTEX_32); + if (res != 1) { + ksft_test_result_fail("futex2_requeue private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_requeue simple succeeds\n"); + } + + + /* + * Create 10 waiters at f1. At futex_requeue, wake 3 and requeue 7. + * At futex_wake, wake INT_MAX (should be exaclty 7). + */ + for (i = 0; i < 10; i++) { + if (pthread_create(&waiter[i], NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + } + + usleep(WAKE_WAIT_US); + + res = futex2_requeue(&r1, &r2, 3, 7, 0, 0); + if (res != 10) { + ksft_test_result_fail("futex2_requeue private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } + + res = futex2_wake(&f2, INT_MAX, FUTEX_32); + if (res != 7) { + ksft_test_result_fail("futex2_requeue private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_requeue succeeds\n"); + } + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/include/futex2test.h b/tools/testing/selftests/futex/include/futex2test.h index 0ed3b20935be96..b9879f1e052304 100644 --- a/tools/testing/selftests/futex/include/futex2test.h +++ b/tools/testing/selftests/futex/include/futex2test.h @@ -94,3 +94,19 @@ static inline int futex2_waitv(volatile struct futex_waitv *waiters, unsigned lo { return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo); } + +/** + * futex2_requeue - Wake futexes at uaddr1 and requeue from uaddr1 to uaddr2 + * @uaddr1: Original address to wake and requeue from + * @uaddr2: Address to requeue to + * @nr_wake: Number of futexes to wake at uaddr1 before requeuing + * @nr_requeue: Number of futexes to requeue from uaddr1 to uaddr2 + * @cmpval: If (uaddr1->uaddr != cmpval), return immediatally + * @flgas: Operation flags + */ +static inline int futex2_requeue(struct futex_requeue *uaddr1, struct futex_requeue *uaddr2, + unsigned int nr_wake, unsigned int nr_requeue, + unsigned int cmpval, unsigned long flags) +{ + return syscall(__NR_futex_requeue, uaddr1, uaddr2, nr_wake, nr_requeue, cmpval, flags); +} From eb5f4b8a680213958859c73b3113b2d5c5bb3ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:02 -0300 Subject: [PATCH 18/48] perf bench: Add futex2 benchmark tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support at the existing futex benchmarking code base to enable futex2 calls. `perf bench` tests can be used not only as a way to measure the performance of implementation, but also as stress testing for the kernel infrastructure. Signed-off-by: André Almeida --- tools/arch/x86/include/asm/unistd_64.h | 12 ++++++ tools/perf/bench/bench.h | 4 ++ tools/perf/bench/futex-hash.c | 24 +++++++++-- tools/perf/bench/futex-requeue.c | 57 ++++++++++++++++++++------ tools/perf/bench/futex-wake-parallel.c | 41 +++++++++++++++--- tools/perf/bench/futex-wake.c | 37 +++++++++++++---- tools/perf/bench/futex.h | 47 +++++++++++++++++++++ tools/perf/builtin-bench.c | 18 ++++++-- 8 files changed, 206 insertions(+), 34 deletions(-) diff --git a/tools/arch/x86/include/asm/unistd_64.h b/tools/arch/x86/include/asm/unistd_64.h index 4205ed4158bfcd..08d7044394a490 100644 --- a/tools/arch/x86/include/asm/unistd_64.h +++ b/tools/arch/x86/include/asm/unistd_64.h @@ -17,3 +17,15 @@ #ifndef __NR_setns #define __NR_setns 308 #endif + +#ifndef __NR_futex_wait +# define __NR_futex_wait 441 +#endif + +#ifndef __NR_futex_wake +# define __NR_futex_wake 442 +#endif + +#ifndef __NR_futex_requeue +# define __NR_futex_requeue 444 +#endif diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h index eac36afab2b39f..12346844b354e4 100644 --- a/tools/perf/bench/bench.h +++ b/tools/perf/bench/bench.h @@ -38,9 +38,13 @@ int bench_mem_memcpy(int argc, const char **argv); int bench_mem_memset(int argc, const char **argv); int bench_mem_find_bit(int argc, const char **argv); int bench_futex_hash(int argc, const char **argv); +int bench_futex2_hash(int argc, const char **argv); int bench_futex_wake(int argc, const char **argv); +int bench_futex2_wake(int argc, const char **argv); int bench_futex_wake_parallel(int argc, const char **argv); +int bench_futex2_wake_parallel(int argc, const char **argv); int bench_futex_requeue(int argc, const char **argv); +int bench_futex2_requeue(int argc, const char **argv); /* pi futexes */ int bench_futex_lock_pi(int argc, const char **argv); int bench_epoll_wait(int argc, const char **argv); diff --git a/tools/perf/bench/futex-hash.c b/tools/perf/bench/futex-hash.c index 915bf3da7ce222..6e62e7708fdebd 100644 --- a/tools/perf/bench/futex-hash.c +++ b/tools/perf/bench/futex-hash.c @@ -34,7 +34,7 @@ static unsigned int nthreads = 0; static unsigned int nsecs = 10; /* amount of futexes per thread */ static unsigned int nfutexes = 1024; -static bool fshared = false, done = false, silent = false; +static bool fshared = false, done = false, silent = false, futex2 = false; static int futex_flag = 0; struct timeval bench__start, bench__end, bench__runtime; @@ -86,7 +86,10 @@ static void *workerfn(void *arg) * such as internal waitqueue handling, thus enlarging * the critical region protected by hb->lock. */ - ret = futex_wait(&w->futex[i], 1234, NULL, futex_flag); + if (!futex2) + ret = futex_wait(&w->futex[i], 1234, NULL, futex_flag); + else + ret = futex2_wait(&w->futex[i], 1234, futex_flag, NULL); if (!silent && (!ret || errno != EAGAIN || errno != EWOULDBLOCK)) warn("Non-expected futex return call"); @@ -117,7 +120,7 @@ static void print_summary(void) (int)bench__runtime.tv_sec); } -int bench_futex_hash(int argc, const char **argv) +static int __bench_futex_hash(int argc, const char **argv) { int ret = 0; cpu_set_t cpuset; @@ -149,7 +152,9 @@ int bench_futex_hash(int argc, const char **argv) if (!worker) goto errmem; - if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG; printf("Run summary [PID %d]: %d threads, each operating on %d [%s] futexes for %d secs.\n\n", @@ -229,3 +234,14 @@ int bench_futex_hash(int argc, const char **argv) errmem: err(EXIT_FAILURE, "calloc"); } + +int bench_futex_hash(int argc, const char **argv) +{ + return __bench_futex_hash(argc, argv); +} + +int bench_futex2_hash(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_hash(argc, argv); +} diff --git a/tools/perf/bench/futex-requeue.c b/tools/perf/bench/futex-requeue.c index 7a15c2e610228f..4c7486fbe92394 100644 --- a/tools/perf/bench/futex-requeue.c +++ b/tools/perf/bench/futex-requeue.c @@ -2,8 +2,8 @@ /* * Copyright (C) 2013 Davidlohr Bueso * - * futex-requeue: Block a bunch of threads on futex1 and requeue them - * on futex2, N at a time. + * futex-requeue: Block a bunch of threads on addr1 and requeue them + * on addr2, N at a time. * * This program is particularly useful to measure the latency of nthread * requeues without waking up any tasks -- thus mimicking a regular futex_wait. @@ -29,7 +29,10 @@ #include #include -static u_int32_t futex1 = 0, futex2 = 0; +static u_int32_t addr1 = 0, addr2 = 0; + +static struct futex_requeue rq1 = { .uaddr = &addr1, .flags = FUTEX_32 }; +static struct futex_requeue rq2 = { .uaddr = &addr2, .flags = FUTEX_32 }; /* * How many tasks to requeue at a time. @@ -38,7 +41,7 @@ static u_int32_t futex1 = 0, futex2 = 0; static unsigned int nrequeue = 1; static pthread_t *worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; static struct stats requeuetime_stats, requeued_stats; @@ -80,7 +83,11 @@ static void *workerfn(void *arg __maybe_unused) pthread_cond_wait(&thread_worker, &thread_lock); pthread_mutex_unlock(&thread_lock); - futex_wait(&futex1, 0, NULL, futex_flag); + if (!futex2) + futex_wait(&addr1, 0, NULL, futex_flag); + else + futex2_wait(&addr1, 0, futex_flag, NULL); + return NULL; } @@ -112,7 +119,7 @@ static void toggle_done(int sig __maybe_unused, done = true; } -int bench_futex_requeue(int argc, const char **argv) +static int __bench_futex_requeue(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -140,15 +147,20 @@ int bench_futex_requeue(int argc, const char **argv) if (!worker) err(EXIT_FAILURE, "calloc"); - if (!fshared) + if (futex2) { + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + rq1.flags |= FUTEX_SHARED_FLAG * fshared; + rq2.flags |= FUTEX_SHARED_FLAG * fshared; + } else if (!fshared) { futex_flag = FUTEX_PRIVATE_FLAG; + } if (nrequeue > nthreads) nrequeue = nthreads; printf("Run summary [PID %d]: Requeuing %d threads (from [%s] %p to %p), " "%d at a time.\n\n", getpid(), nthreads, - fshared ? "shared":"private", &futex1, &futex2, nrequeue); + fshared ? "shared":"private", &addr1, &addr2, nrequeue); init_stats(&requeued_stats); init_stats(&requeuetime_stats); @@ -177,11 +189,15 @@ int bench_futex_requeue(int argc, const char **argv) gettimeofday(&start, NULL); while (nrequeued < nthreads) { /* - * Do not wakeup any tasks blocked on futex1, allowing + * Do not wakeup any tasks blocked on addr1, allowing * us to really measure futex_wait functionality. */ - nrequeued += futex_cmp_requeue(&futex1, 0, &futex2, 0, - nrequeue, futex_flag); + if (!futex2) + nrequeued += futex_cmp_requeue(&addr1, 0, &addr2, + 0, nrequeue, futex_flag); + else + nrequeued += futex2_requeue(&rq1, &rq2, + 0, nrequeue, 0, 0); } gettimeofday(&end, NULL); @@ -195,8 +211,12 @@ int bench_futex_requeue(int argc, const char **argv) j + 1, nrequeued, nthreads, runtime.tv_usec / (double)USEC_PER_MSEC); } - /* everybody should be blocked on futex2, wake'em up */ - nrequeued = futex_wake(&futex2, nrequeued, futex_flag); + /* everybody should be blocked on addr2, wake'em up */ + if (!futex2) + nrequeued = futex_wake(&addr2, nrequeued, futex_flag); + else + nrequeued = futex2_wake(&addr2, nrequeued, futex_flag); + if (nthreads != nrequeued) warnx("couldn't wakeup all tasks (%d/%d)", nrequeued, nthreads); @@ -221,3 +241,14 @@ int bench_futex_requeue(int argc, const char **argv) usage_with_options(bench_futex_requeue_usage, options); exit(EXIT_FAILURE); } + +int bench_futex_requeue(int argc, const char **argv) +{ + return __bench_futex_requeue(argc, argv); +} + +int bench_futex2_requeue(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_requeue(argc, argv); +} diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c index cd2b81a845acb0..8a89c6ab95411a 100644 --- a/tools/perf/bench/futex-wake-parallel.c +++ b/tools/perf/bench/futex-wake-parallel.c @@ -17,6 +17,12 @@ int bench_futex_wake_parallel(int argc __maybe_unused, const char **argv __maybe pr_err("%s: pthread_barrier_t unavailable, disabling this test...\n", __func__); return 0; } + +int bench_futex2_wake_parallel(int argc __maybe_unused, const char **argv __maybe_unused) +{ + pr_err("%s: pthread_barrier_t unavailable, disabling this test...\n", __func__); + return 0; +} #else /* HAVE_PTHREAD_BARRIER */ /* For the CLR_() macros */ #include @@ -48,7 +54,7 @@ static unsigned int nwakes = 1; static u_int32_t futex = 0; static pthread_t *blocked_worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static unsigned int nblocked_threads = 0, nwaking_threads = 0; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; @@ -79,7 +85,11 @@ static void *waking_workerfn(void *arg) gettimeofday(&start, NULL); - waker->nwoken = futex_wake(&futex, nwakes, futex_flag); + if (!futex2) + waker->nwoken = futex_wake(&futex, nwakes, futex_flag); + else + waker->nwoken = futex2_wake(&futex, nwakes, futex_flag); + if (waker->nwoken != nwakes) warnx("couldn't wakeup all tasks (%d/%d)", waker->nwoken, nwakes); @@ -130,8 +140,13 @@ static void *blocked_workerfn(void *arg __maybe_unused) pthread_mutex_unlock(&thread_lock); while (1) { /* handle spurious wakeups */ - if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR) - break; + if (!futex2) { + if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR) + break; + } else { + if (futex2_wait(&futex, 0, futex_flag, NULL) != EINTR) + break; + } } pthread_exit(NULL); @@ -218,7 +233,7 @@ static void toggle_done(int sig __maybe_unused, done = true; } -int bench_futex_wake_parallel(int argc, const char **argv) +static int __bench_futex_wake_parallel(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -262,7 +277,9 @@ int bench_futex_wake_parallel(int argc, const char **argv) if (!blocked_worker) err(EXIT_FAILURE, "calloc"); - if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG; printf("Run summary [PID %d]: blocking on %d threads (at [%s] " @@ -322,4 +339,16 @@ int bench_futex_wake_parallel(int argc, const char **argv) free(blocked_worker); return ret; } + +int bench_futex_wake_parallel(int argc, const char **argv) +{ + return __bench_futex_wake_parallel(argc, argv); +} + +int bench_futex2_wake_parallel(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_wake_parallel(argc, argv); +} + #endif /* HAVE_PTHREAD_BARRIER */ diff --git a/tools/perf/bench/futex-wake.c b/tools/perf/bench/futex-wake.c index 2dfcef3e371e4b..be4481f5ee98ba 100644 --- a/tools/perf/bench/futex-wake.c +++ b/tools/perf/bench/futex-wake.c @@ -39,7 +39,7 @@ static u_int32_t futex1 = 0; static unsigned int nwakes = 1; pthread_t *worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; static struct stats waketime_stats, wakeup_stats; @@ -69,8 +69,13 @@ static void *workerfn(void *arg __maybe_unused) pthread_mutex_unlock(&thread_lock); while (1) { - if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR) - break; + if (!futex2) { + if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR) + break; + } else { + if (futex2_wait(&futex1, 0, futex_flag, NULL) != EINTR) + break; + } } pthread_exit(NULL); @@ -118,7 +123,7 @@ static void toggle_done(int sig __maybe_unused, done = true; } -int bench_futex_wake(int argc, const char **argv) +static int __bench_futex_wake(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -148,7 +153,9 @@ int bench_futex_wake(int argc, const char **argv) if (!worker) err(EXIT_FAILURE, "calloc"); - if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG; printf("Run summary [PID %d]: blocking on %d threads (at [%s] futex %p), " @@ -180,9 +187,14 @@ int bench_futex_wake(int argc, const char **argv) /* Ok, all threads are patiently blocked, start waking folks up */ gettimeofday(&start, NULL); - while (nwoken != nthreads) - nwoken += futex_wake(&futex1, nwakes, futex_flag); + while (nwoken != nthreads) { + if (!futex2) + nwoken += futex_wake(&futex1, nwakes, futex_flag); + else + nwoken += futex2_wake(&futex1, nwakes, futex_flag); + } gettimeofday(&end, NULL); + timersub(&end, &start, &runtime); update_stats(&wakeup_stats, nwoken); @@ -212,3 +224,14 @@ int bench_futex_wake(int argc, const char **argv) free(worker); return ret; } + +int bench_futex_wake(int argc, const char **argv) +{ + return __bench_futex_wake(argc, argv); +} + +int bench_futex2_wake(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_wake(argc, argv); +} diff --git a/tools/perf/bench/futex.h b/tools/perf/bench/futex.h index 31b53cc7d5bc77..6b2213cf3f645b 100644 --- a/tools/perf/bench/futex.h +++ b/tools/perf/bench/futex.h @@ -86,4 +86,51 @@ futex_cmp_requeue(u_int32_t *uaddr, u_int32_t val, u_int32_t *uaddr2, int nr_wak return futex(uaddr, FUTEX_CMP_REQUEUE, nr_wake, nr_requeue, uaddr2, val, opflags); } + +/** + * futex2_wait - Wait at uaddr if *uaddr == val, until timo. + * @uaddr: User address to wait for + * @val: Expected value at uaddr + * @flags: Operation options + * @timo: Optional timeout + * + * Return: 0 on success, error code otherwise + */ +static inline int futex2_wait(volatile void *uaddr, unsigned long val, + unsigned long flags, struct timespec *timo) +{ + return syscall(__NR_futex_wait, uaddr, val, flags, timo); +} + +/** + * futex2_wake - Wake a number of waiters waiting at uaddr + * @uaddr: Address to wake + * @nr: Number of waiters to wake + * @flags: Operation options + * + * Return: number of waked futexes + */ +static inline int futex2_wake(volatile void *uaddr, unsigned int nr, unsigned long flags) +{ + return syscall(__NR_futex_wake, uaddr, nr, flags); +} + +/** + * futex2_requeue - Requeue waiters from an address to another one + * @uaddr1: Address where waiters are currently waiting on + * @uaddr2: New address to wait + * @nr_wake: Number of waiters at uaddr1 to be wake + * @nr_requeue: After waking nr_wake, number of waiters to be requeued + * @cmpval: Expected value at uaddr1 + * @flags: Operation options + * + * Return: waked futexes + requeued futexes at uaddr1 + */ +static inline int futex2_requeue(volatile struct futex_requeue *uaddr1, + volatile struct futex_requeue *uaddr2, + unsigned int nr_wake, unsigned int nr_requeue, + unsigned int cmpval, unsigned long flags) +{ + return syscall(__NR_futex_requeue, uaddr1, uaddr2, nr_wake, nr_requeue, cmpval, flags); +} #endif /* _FUTEX_H */ diff --git a/tools/perf/builtin-bench.c b/tools/perf/builtin-bench.c index 62a7b7420a4485..e41a95ad2db64a 100644 --- a/tools/perf/builtin-bench.c +++ b/tools/perf/builtin-bench.c @@ -12,10 +12,11 @@ * * sched ... scheduler and IPC performance * syscall ... System call performance - * mem ... memory access performance - * numa ... NUMA scheduling and MM performance - * futex ... Futex performance - * epoll ... Event poll performance + * mem ... memory access performance + * numa ... NUMA scheduling and MM performance + * futex ... Futex performance + * futex2 ... Futex2 performance + * epoll ... Event poll performance */ #include #include "builtin.h" @@ -75,6 +76,14 @@ static struct bench futex_benchmarks[] = { { NULL, NULL, NULL } }; +static struct bench futex2_benchmarks[] = { + { "hash", "Benchmark for futex2 hash table", bench_futex2_hash }, + { "wake", "Benchmark for futex2 wake calls", bench_futex2_wake }, + { "wake-parallel", "Benchmark for parallel futex2 wake calls", bench_futex2_wake_parallel }, + { "requeue", "Benchmark for futex2 requeue calls", bench_futex2_requeue }, + { NULL, NULL, NULL } +}; + #ifdef HAVE_EVENTFD_SUPPORT static struct bench epoll_benchmarks[] = { { "wait", "Benchmark epoll concurrent epoll_waits", bench_epoll_wait }, @@ -105,6 +114,7 @@ static struct collection collections[] = { { "numa", "NUMA scheduling and MM benchmarks", numa_benchmarks }, #endif {"futex", "Futex stressing benchmarks", futex_benchmarks }, + {"futex2", "Futex2 stressing benchmarks", futex2_benchmarks }, #ifdef HAVE_EVENTFD_SUPPORT {"epoll", "Epoll stressing benchmarks", epoll_benchmarks }, #endif From 768644ffcaada56bdcb652d57f82eae05d038f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:02 -0300 Subject: [PATCH 19/48] kernel: Enable waitpid() for futex2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To make pthreads works as expected if they are using futex2, wake clear_child_tid with futex2 as well. This is make applications that uses waitpid() (and clone(CLONE_CHILD_SETTID)) wake while waiting for the child to terminate. Given that apps should not mix futex() and futex2(), any correct app will trigger a harmless noop wakeup on the interface that it isn't using. Signed-off-by: André Almeida --- This commit is here for the intend to show what we need to do in order to get a full NPTL working on top of futex2. It should be merged after we talk to glibc folks on the details around the futex_wait() side. For instance, we could use this as an opportunity to use private futexes or 8bit sized futexes, but both sides need to use the exactly same flags. --- include/linux/syscalls.h | 2 ++ kernel/fork.c | 2 ++ kernel/futex2.c | 30 ++++++++++++++++++------------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index d80eba8a43e544..6021709f4291cb 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1306,6 +1306,8 @@ int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); int compat_ksys_ipc(u32 call, int first, int second, u32 third, u32 ptr, u32 fifth); +long ksys_futex_wake(void __user *uaddr, unsigned long nr_wake, + unsigned int flags); /* * The following kernel syscall equivalents are just wrappers to fs-internal diff --git a/kernel/fork.c b/kernel/fork.c index 7c044d377926cc..fd7cd688c673d5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1314,6 +1314,8 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm) put_user(0, tsk->clear_child_tid); do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1, NULL, NULL, 0, 0); + ksys_futex_wake(tsk->clear_child_tid, 1, + FUTEX_32 | FUTEX_SHARED_FLAG); } tsk->clear_child_tid = NULL; } diff --git a/kernel/futex2.c b/kernel/futex2.c index 49f474bddb76cf..6aa04dddca21be 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -928,18 +928,8 @@ static inline bool futex_match(struct futex_key key1, struct futex_key key2) key1.offset == key2.offset); } -/** - * sys_futex_wake - Wake a number of futexes waiting on an address - * @uaddr: Address of futex to be woken up - * @nr_wake: Number of futexes waiting in uaddr to be woken up - * @flags: Flags for size and shared - * - * Wake `nr_wake` threads waiting at uaddr. - * - * Returns the number of woken threads on success, error code otherwise. - */ -SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, - unsigned int, flags) +long ksys_futex_wake(void __user *uaddr, unsigned long nr_wake, + unsigned int flags) { bool shared = (flags & FUTEX_SHARED_FLAG) ? true : false; unsigned int size = flags & FUTEX_SIZE_MASK; @@ -976,6 +966,22 @@ SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, return ret; } +/** + * sys_futex_wake - Wake a number of futexes waiting on an address + * @uaddr: Address of futex to be woken up + * @nr_wake: Number of futexes waiting in uaddr to be woken up + * @flags: Flags for size and shared + * + * Wake `nr_wake` threads waiting at uaddr. + * + * Returns the number of woken threads on success, error code otherwise. + */ +SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake, + unsigned int, flags) +{ + return ksys_futex_wake(uaddr, nr_wake, flags); +} + static void futex_double_unlock(struct futex_bucket *b1, struct futex_bucket *b2) { spin_unlock(&b1->lock); From e1f13dd0a7bfcb2de9e36991d58ec9a48a109f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Fri, 5 Feb 2021 10:34:02 -0300 Subject: [PATCH 20/48] futex2: Add sysfs entry for syscall numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the course of futex2 development, it will be rebased on top of different kernel releases, and the syscall number can change in this process. Expose futex2 syscall number via sysfs so tools that are experimenting with futex2 (like Proton/Wine) can test it and set the syscall number at runtime, rather than setting it at compilation time. Signed-off-by: André Almeida --- kernel/futex2.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/kernel/futex2.c b/kernel/futex2.c index 6aa04dddca21be..c9545ff346a78b 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -1212,6 +1212,48 @@ SYSCALL_DEFINE6(futex_requeue, struct futex_requeue __user *, uaddr1, return __futex_requeue(rq1, rq2, nr_wake, nr_requeue, cmpval, shared1, shared2); } +static ssize_t wait_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", __NR_futex_wait); + +} +static struct kobj_attribute futex2_wait_attr = __ATTR_RO(wait); + +static ssize_t wake_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", __NR_futex_wake); + +} +static struct kobj_attribute futex2_wake_attr = __ATTR_RO(wake); + +static ssize_t waitv_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", __NR_futex_waitv); + +} +static struct kobj_attribute futex2_waitv_attr = __ATTR_RO(waitv); + +static struct attribute *futex2_sysfs_attrs[] = { + &futex2_wait_attr.attr, + &futex2_wake_attr.attr, + &futex2_waitv_attr.attr, + NULL, +}; + +static const struct attribute_group futex2_sysfs_attr_group = { + .attrs = futex2_sysfs_attrs, + .name = "futex2", +}; + +static int __init futex2_sysfs_init(void) +{ + return sysfs_create_group(kernel_kobj, &futex2_sysfs_attr_group); +} +subsys_initcall(futex2_sysfs_init); + static int __init futex2_init(void) { int i; From cc823dc48c24083c2fff1327134bcada98246faf Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 10:50:45 -0600 Subject: [PATCH 21/48] winesync: Introduce the winesync driver and character device. Signed-off-by: Kai Krakow --- drivers/misc/Kconfig | 11 +++++++ drivers/misc/Makefile | 1 + drivers/misc/winesync.c | 64 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 drivers/misc/winesync.c diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index fafa8b0d809960..25bf30ab958a7d 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -466,6 +466,17 @@ config HISI_HIKEY_USB switching between the dual-role USB-C port and the USB-A host ports using only one USB controller. +config WINESYNC + tristate "Synchronization primitives for Wine" + help + This module provides kernel support for synchronization primitives + used by Wine. It is not a hardware driver. + + To compile this driver as a module, choose M here: the + module will be called winesync. + + If unsure, say N. + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index d23231e7333037..8b1d4ae4ed1bb6 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -57,3 +57,4 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ obj-$(CONFIG_UACCE) += uacce/ obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o +obj-$(CONFIG_WINESYNC) += winesync.o diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c new file mode 100644 index 00000000000000..111f33c5676e6c --- /dev/null +++ b/drivers/misc/winesync.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * winesync.c - Kernel driver for Wine synchronization primitives + * + * Copyright (C) 2021 Zebediah Figura + */ + +#include +#include +#include + +#define WINESYNC_NAME "winesync" + +static int winesync_char_open(struct inode *inode, struct file *file) +{ + return nonseekable_open(inode, file); +} + +static int winesync_char_release(struct inode *inode, struct file *file) +{ + return 0; +} + +static long winesync_char_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + switch (cmd) { + default: + return -ENOSYS; + } +} + +static const struct file_operations winesync_fops = { + .owner = THIS_MODULE, + .open = winesync_char_open, + .release = winesync_char_release, + .unlocked_ioctl = winesync_char_ioctl, + .compat_ioctl = winesync_char_ioctl, + .llseek = no_llseek, +}; + +static struct miscdevice winesync_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = WINESYNC_NAME, + .fops = &winesync_fops, +}; + +static int __init winesync_init(void) +{ + return misc_register(&winesync_misc); +} + +static void __exit winesync_exit(void) +{ + misc_deregister(&winesync_misc); +} + +module_init(winesync_init); +module_exit(winesync_exit); + +MODULE_AUTHOR("Zebediah Figura"); +MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("devname:" WINESYNC_NAME); From bed06a9f52038a2c1dbdbdce3bfb3b4410bcb287 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 10:57:06 -0600 Subject: [PATCH 22/48] winesync: Reserve a minor device number and ioctl range. Signed-off-by: Kai Krakow --- Documentation/admin-guide/devices.txt | 3 ++- Documentation/userspace-api/ioctl/ioctl-number.rst | 2 ++ drivers/misc/winesync.c | 3 ++- include/linux/miscdevice.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt index 63fd4e6a014bcc..dc9a30b5f1a21e 100644 --- a/Documentation/admin-guide/devices.txt +++ b/Documentation/admin-guide/devices.txt @@ -376,8 +376,9 @@ 240 = /dev/userio Serio driver testing device 241 = /dev/vhost-vsock Host kernel driver for virtio vsock 242 = /dev/rfkill Turning off radio transmissions (rfkill) + 243 = /dev/winesync Wine synchronization primitive device - 243-254 Reserved for local use + 244-254 Reserved for local use 255 Reserved for MISC_DYNAMIC_MINOR 11 char Raw keyboard device (Linux/SPARC only) diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 55a2d9b2ce33c4..1993277f6153ac 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -361,6 +361,8 @@ Code Seq# Include File Comments 0xF6 all LTTng Linux Trace Toolkit Next Generation +0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives + 0xFD all linux/dm-ioctl.h 0xFE all linux/isst_if.h ==== ===== ======================================================= ================================================================ diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 111f33c5676e6c..85cb6ccaa077db 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -40,7 +40,7 @@ static const struct file_operations winesync_fops = { }; static struct miscdevice winesync_misc = { - .minor = MISC_DYNAMIC_MINOR, + .minor = WINESYNC_MINOR, .name = WINESYNC_NAME, .fops = &winesync_fops, }; @@ -62,3 +62,4 @@ MODULE_AUTHOR("Zebediah Figura"); MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); MODULE_LICENSE("GPL"); MODULE_ALIAS("devname:" WINESYNC_NAME); +MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h index 0676f18093f922..350aecfcfb2908 100644 --- a/include/linux/miscdevice.h +++ b/include/linux/miscdevice.h @@ -71,6 +71,7 @@ #define USERIO_MINOR 240 #define VHOST_VSOCK_MINOR 241 #define RFKILL_MINOR 242 +#define WINESYNC_MINOR 243 #define MISC_DYNAMIC_MINOR 255 struct device; From a2b1b69c98c6a5de595e5c7a45ed660e2fe8bde3 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:15:39 -0600 Subject: [PATCH 23/48] winesync: Introduce WINESYNC_IOC_CREATE_SEM and WINESYNC_IOC_DELETE. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 128 ++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 25 +++++++ 2 files changed, 153 insertions(+) create mode 100644 include/uapi/linux/winesync.h diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 85cb6ccaa077db..37622e1eaa29b0 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -6,25 +6,153 @@ */ #include +#include #include #include +#include +#include +#include #define WINESYNC_NAME "winesync" +enum winesync_type { + WINESYNC_TYPE_SEM, +}; + +struct winesync_obj { + struct kref refcount; + + enum winesync_type type; + + union { + struct { + __u32 count; + __u32 max; + } sem; + } u; +}; + +struct winesync_device { + struct mutex table_lock; + struct idr objects; +}; + +static void destroy_obj(struct kref *ref) +{ + struct winesync_obj *obj; + + obj = container_of(ref, struct winesync_obj, refcount); + kfree(obj); +} + +static void put_obj(struct winesync_obj *obj) +{ + kref_put(&obj->refcount, destroy_obj); +} + static int winesync_char_open(struct inode *inode, struct file *file) { + struct winesync_device *dev; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + idr_init(&dev->objects); + mutex_init(&dev->table_lock); + + file->private_data = dev; return nonseekable_open(inode, file); } static int winesync_char_release(struct inode *inode, struct file *file) { + struct winesync_device *dev = file->private_data; + struct winesync_obj *obj; + int id; + + mutex_lock(&dev->table_lock); + idr_for_each_entry(&dev->objects, obj, id) { + idr_remove(&dev->objects, id); + synchronize_rcu(); + put_obj(obj); + } + mutex_unlock(&dev->table_lock); + + kfree(dev); + + return 0; +} + +static void winesync_init_obj(struct winesync_obj *obj) +{ + kref_init(&obj->refcount); +} + +static int winesync_create_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (args.count > args.max) + return -EINVAL; + + sem = kzalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return -ENOMEM; + + winesync_init_obj(sem); + sem->type = WINESYNC_TYPE_SEM; + sem->u.sem.count = args.count; + sem->u.sem.max = args.max; + + mutex_lock(&dev->table_lock); + ret = idr_alloc(&dev->objects, sem, 0, 0, GFP_KERNEL); + mutex_unlock(&dev->table_lock); + + if (ret < 0) { + kfree(sem); + return ret; + } + + return put_user(ret, &user_args->sem); +} + +static int winesync_delete(struct winesync_device *dev, void __user *argp) +{ + struct winesync_obj *obj; + __u32 id; + + if (get_user(id, (__u32 __user *)argp)) + return -EFAULT; + + mutex_lock(&dev->table_lock); + obj = idr_remove(&dev->objects, id); + mutex_unlock(&dev->table_lock); + + if (!obj) + return -EINVAL; + + put_obj(obj); return 0; } static long winesync_char_ioctl(struct file *file, unsigned int cmd, unsigned long parm) { + struct winesync_device *dev = file->private_data; + void __user *argp = (void __user *)parm; + switch (cmd) { + case WINESYNC_IOC_CREATE_SEM: + return winesync_create_sem(dev, argp); + case WINESYNC_IOC_DELETE: + return winesync_delete(dev, argp); default: return -ENOSYS; } diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h new file mode 100644 index 00000000000000..9096df8e2d9926 --- /dev/null +++ b/include/uapi/linux/winesync.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Kernel support for Wine synchronization primitives + * + * Copyright (C) 2021 Zebediah Figura + */ + +#ifndef __LINUX_WINESYNC_H +#define __LINUX_WINESYNC_H + +#include + +struct winesync_sem_args { + __u32 sem; + __u32 count; + __u32 max; +}; + +#define WINESYNC_IOC_BASE 0xf7 + +#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ + struct winesync_sem_args) +#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) + +#endif From 7dfccfc3d633e8d52a02bc25b156a57d3524d458 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:22:42 -0600 Subject: [PATCH 24/48] winesync: Introduce WINESYNC_PUT_SEM. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 65 +++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 2 ++ 2 files changed, 67 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 37622e1eaa29b0..f8e1d0d574c33a 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -21,9 +21,11 @@ enum winesync_type { struct winesync_obj { struct kref refcount; + spinlock_t lock; enum winesync_type type; + /* The following fields are protected by the object lock. */ union { struct { __u32 count; @@ -37,6 +39,19 @@ struct winesync_device { struct idr objects; }; +static struct winesync_obj *get_obj(struct winesync_device *dev, int id) +{ + struct winesync_obj *obj; + + rcu_read_lock(); + obj = idr_find(&dev->objects, id); + if (obj && !kref_get_unless_zero(&obj->refcount)) + obj = NULL; + rcu_read_unlock(); + + return obj; +} + static void destroy_obj(struct kref *ref) { struct winesync_obj *obj; @@ -87,6 +102,7 @@ static int winesync_char_release(struct inode *inode, struct file *file) static void winesync_init_obj(struct winesync_obj *obj) { kref_init(&obj->refcount); + spin_lock_init(&obj->lock); } static int winesync_create_sem(struct winesync_device *dev, void __user *argp) @@ -142,6 +158,53 @@ static int winesync_delete(struct winesync_device *dev, void __user *argp) return 0; } +/* + * Actually change the semaphore state, returning -EOVERFLOW if it is made + * invalid. + */ +static int put_sem_state(struct winesync_obj *sem, __u32 count) +{ + lockdep_assert_held(&sem->lock); + + if (sem->u.sem.count + count < sem->u.sem.count || + sem->u.sem.count + count > sem->u.sem.max) + return -EOVERFLOW; + + sem->u.sem.count += count; + return 0; +} + +static int winesync_put_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + __u32 prev_count; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + sem = get_obj(dev, args.sem); + if (!sem) + return -EINVAL; + if (sem->type != WINESYNC_TYPE_SEM) { + put_obj(sem); + return -EINVAL; + } + + spin_lock(&sem->lock); + ret = put_sem_state(sem, args.count); + spin_unlock(&sem->lock); + + put_obj(sem); + + if (!ret && put_user(prev_count, &user_args->count)) + ret = -EFAULT; + + return ret; +} + static long winesync_char_ioctl(struct file *file, unsigned int cmd, unsigned long parm) { @@ -153,6 +216,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_create_sem(dev, argp); case WINESYNC_IOC_DELETE: return winesync_delete(dev, argp); + case WINESYNC_IOC_PUT_SEM: + return winesync_put_sem(dev, argp); default: return -ENOSYS; } diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 9096df8e2d9926..6447c29fc41977 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -20,6 +20,8 @@ struct winesync_sem_args { #define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ struct winesync_sem_args) +#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ + struct winesync_sem_args) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #endif From f6e1c4189d1d5e22a90ffe3764d115cc6c841b3a Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:31:44 -0600 Subject: [PATCH 25/48] winesync: Introduce WINESYNC_IOC_WAIT_ANY. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 222 ++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 11 ++ 2 files changed, 233 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index f8e1d0d574c33a..b3a912dff92227 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -23,6 +23,8 @@ struct winesync_obj { struct kref refcount; spinlock_t lock; + struct list_head any_waiters; + enum winesync_type type; /* The following fields are protected by the object lock. */ @@ -34,6 +36,28 @@ struct winesync_obj { } u; }; +struct winesync_q_entry { + struct list_head node; + struct winesync_q *q; + struct winesync_obj *obj; + __u32 index; +}; + +struct winesync_q { + struct task_struct *task; + __u32 owner; + + /* + * Protected via atomic_cmpxchg(). Only the thread that wins the + * compare-and-swap may actually change object states and wake this + * task. + */ + atomic_t signaled; + + __u32 count; + struct winesync_q_entry entries[]; +}; + struct winesync_device { struct mutex table_lock; struct idr objects; @@ -103,6 +127,26 @@ static void winesync_init_obj(struct winesync_obj *obj) { kref_init(&obj->refcount); spin_lock_init(&obj->lock); + INIT_LIST_HEAD(&obj->any_waiters); +} + +static void try_wake_any_sem(struct winesync_obj *sem) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&sem->lock); + + list_for_each_entry(entry, &sem->any_waiters, node) { + struct winesync_q *q = entry->q; + + if (!sem->u.sem.count) + break; + + if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + sem->u.sem.count--; + wake_up_process(q->task); + } + } } static int winesync_create_sem(struct winesync_device *dev, void __user *argp) @@ -195,6 +239,8 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) spin_lock(&sem->lock); ret = put_sem_state(sem, args.count); + if (!ret) + try_wake_any_sem(sem); spin_unlock(&sem->lock); put_obj(sem); @@ -205,6 +251,180 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) return ret; } +static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) +{ + int ret = 0; + + do { + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + set_current_state(TASK_INTERRUPTIBLE); + if (atomic_read(&q->signaled) != -1) { + ret = 0; + break; + } + ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); + } while (ret < 0); + __set_current_state(TASK_RUNNING); + + return ret; +} + +/* + * Allocate and initialize most of the winesync_q structure, but do not queue us + * yet. Also, calculate the relative timeout in jiffies. + */ +static int setup_wait(struct winesync_device *dev, + const struct winesync_wait_args *args, + ktime_t *ret_timeout, struct winesync_q **ret_q) +{ + const __u32 count = args->count; + struct winesync_q *q; + ktime_t timeout = 0; + __u32 *ids; + __u32 i, j; + + if (args->timeout) { + struct timespec64 to; + + if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) + return -EFAULT; + if (!timespec64_valid(&to)) + return -EINVAL; + + timeout = timespec64_to_ns(&to); + } + + ids = kmalloc_array(args->count, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; + if (copy_from_user(ids, u64_to_user_ptr(args->objs), + array_size(args->count, sizeof(*ids)))) { + kfree(ids); + return -EFAULT; + } + + q = kmalloc(struct_size(q, entries, count), GFP_KERNEL); + if (!q) { + kfree(ids); + return -ENOMEM; + } + q->task = current; + q->owner = args->owner; + atomic_set(&q->signaled, -1); + q->count = count; + + for (i = 0; i < count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = get_obj(dev, ids[i]); + + if (!obj) + goto err; + + entry->obj = obj; + entry->q = q; + entry->index = i; + } + + kfree(ids); + + *ret_q = q; + *ret_timeout = timeout; + return 0; + +err: + for (j = 0; j < i; j++) + put_obj(q->entries[j].obj); + kfree(ids); + kfree(q); + return -EINVAL; +} + +static void try_wake_any_obj(struct winesync_obj *obj) +{ + switch (obj->type) { + case WINESYNC_TYPE_SEM: + try_wake_any_sem(obj); + break; + } +} + +static int winesync_wait_any(struct winesync_device *dev, void __user *argp) +{ + struct winesync_wait_args args; + struct winesync_q *q; + ktime_t timeout; + __u32 i; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (!args.owner) + return -EINVAL; + + ret = setup_wait(dev, &args, &timeout, &q); + if (ret < 0) + return ret; + + /* queue ourselves */ + + for (i = 0; i < args.count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = q->entries[i].obj; + + spin_lock(&obj->lock); + list_add_tail(&entry->node, &obj->any_waiters); + spin_unlock(&obj->lock); + } + + /* check if we are already signaled */ + + for (i = 0; i < args.count; i++) { + struct winesync_obj *obj = q->entries[i].obj; + + if (atomic_read(&q->signaled) != -1) + break; + + spin_lock(&obj->lock); + try_wake_any_obj(obj); + spin_unlock(&obj->lock); + } + + /* sleep */ + + ret = winesync_schedule(q, args.timeout ? &timeout : NULL); + + /* and finally, unqueue */ + + for (i = 0; i < args.count; i++) { + struct winesync_obj *obj = q->entries[i].obj; + + spin_lock(&obj->lock); + list_del(&q->entries[i].node); + spin_unlock(&obj->lock); + + put_obj(obj); + } + + if (atomic_read(&q->signaled) != -1) { + struct winesync_wait_args __user *user_args = argp; + + /* even if we caught a signal, we need to communicate success */ + ret = 0; + + if (put_user(atomic_read(&q->signaled), &user_args->index)) + ret = -EFAULT; + } else if (!ret) { + ret = -ETIMEDOUT; + } + + kfree(q); + return ret; +} + static long winesync_char_ioctl(struct file *file, unsigned int cmd, unsigned long parm) { @@ -218,6 +438,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_delete(dev, argp); case WINESYNC_IOC_PUT_SEM: return winesync_put_sem(dev, argp); + case WINESYNC_IOC_WAIT_ANY: + return winesync_wait_any(dev, argp); default: return -ENOSYS; } diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 6447c29fc41977..c9e18310e77f0a 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -16,6 +16,15 @@ struct winesync_sem_args { __u32 max; }; +struct winesync_wait_args { + __u64 timeout; + __u64 objs; + __u32 count; + __u32 owner; + __u32 index; + __u32 pad; +}; + #define WINESYNC_IOC_BASE 0xf7 #define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ @@ -23,5 +32,7 @@ struct winesync_sem_args { #define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ struct winesync_sem_args) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) +#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ + struct winesync_wait_args) #endif From ef416934b2155c278f5b17187a6938534e81954f Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:36:09 -0600 Subject: [PATCH 26/48] winesync: Introduce WINESYNC_IOC_WAIT_ALL. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 238 +++++++++++++++++++++++++++++++++- include/uapi/linux/winesync.h | 2 + 2 files changed, 233 insertions(+), 7 deletions(-) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index b3a912dff92227..fe87082f80a728 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -23,7 +23,36 @@ struct winesync_obj { struct kref refcount; spinlock_t lock; + /* + * any_waiters is protected by the object lock, but all_waiters is + * protected by the device wait_all_lock. + */ struct list_head any_waiters; + struct list_head all_waiters; + + /* + * Hint describing how many tasks are queued on this object in a + * wait-all operation. + * + * Any time we do a wake, we may need to wake "all" waiters as well as + * "any" waiters. In order to atomically wake "all" waiters, we must + * lock all of the objects, and that means grabbing the wait_all_lock + * below (and, due to lock ordering rules, before locking this object). + * However, wait-all is a rare operation, and grabbing the wait-all + * lock for every wake would create unnecessary contention. Therefore we + * first check whether all_hint is one, and, if it is, we skip trying + * to wake "all" waiters. + * + * This "refcount" isn't protected by any lock. It might change during + * the course of a wake, but there's no meaningful race there; it's only + * a hint. + * + * Use refcount_t rather than atomic_t to take advantage of saturation. + * This does mean that the "no waiters" case is signified by all_hint + * being one, rather than zero (otherwise we would get spurious + * warnings). + */ + refcount_t all_hint; enum winesync_type type; @@ -54,11 +83,24 @@ struct winesync_q { */ atomic_t signaled; + bool all; __u32 count; struct winesync_q_entry entries[]; }; struct winesync_device { + /* + * Wait-all operations must atomically grab all objects, and be totally + * ordered with respect to each other and wait-any operations. If one + * thread is trying to acquire several objects, another thread cannot + * touch the object at the same time. + * + * We achieve this by grabbing multiple object locks at the same time. + * However, this creates a lock ordering problem. To solve that problem, + * wait_all_lock is taken first whenever multiple objects must be locked + * at the same time. + */ + spinlock_t wait_all_lock; struct mutex table_lock; struct idr objects; }; @@ -98,6 +140,7 @@ static int winesync_char_open(struct inode *inode, struct file *file) return -ENOMEM; idr_init(&dev->objects); + spin_lock_init(&dev->wait_all_lock); mutex_init(&dev->table_lock); file->private_data = dev; @@ -126,8 +169,82 @@ static int winesync_char_release(struct inode *inode, struct file *file) static void winesync_init_obj(struct winesync_obj *obj) { kref_init(&obj->refcount); + refcount_set(&obj->all_hint, 1); spin_lock_init(&obj->lock); INIT_LIST_HEAD(&obj->any_waiters); + INIT_LIST_HEAD(&obj->all_waiters); +} + +static bool is_signaled(struct winesync_obj *obj, __u32 owner) +{ + lockdep_assert_held(&obj->lock); + + switch (obj->type) { + case WINESYNC_TYPE_SEM: + return !!obj->u.sem.count; + } + + WARN(1, "bad object type %#x\n", obj->type); + return false; +} + +/* + * "locked_obj" is an optional pointer to an object which is already locked and + * should not be locked again. This is necessary so that changing an object's + * state and waking it can be a single atomic operation. + */ +static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, + struct winesync_obj *locked_obj) +{ + __u32 count = q->count; + bool can_wake = true; + __u32 i; + + lockdep_assert_held(&dev->wait_all_lock); + if (locked_obj) + lockdep_assert_held(&locked_obj->lock); + + for (i = 0; i < count; i++) { + if (q->entries[i].obj != locked_obj) + spin_lock(&q->entries[i].obj->lock); + } + + for (i = 0; i < count; i++) { + if (!is_signaled(q->entries[i].obj, q->owner)) { + can_wake = false; + break; + } + } + + if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { + for (i = 0; i < count; i++) { + struct winesync_obj *obj = q->entries[i].obj; + + switch (obj->type) { + case WINESYNC_TYPE_SEM: + obj->u.sem.count--; + break; + } + } + wake_up_process(q->task); + } + + for (i = 0; i < count; i++) { + if (q->entries[i].obj != locked_obj) + spin_unlock(&q->entries[i].obj->lock); + } +} + +static void try_wake_all_obj(struct winesync_device *dev, + struct winesync_obj *obj) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&dev->wait_all_lock); + lockdep_assert_held(&obj->lock); + + list_for_each_entry(entry, &obj->all_waiters, node) + try_wake_all(dev, entry->q, obj); } static void try_wake_any_sem(struct winesync_obj *sem) @@ -237,11 +354,29 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) return -EINVAL; } - spin_lock(&sem->lock); - ret = put_sem_state(sem, args.count); - if (!ret) - try_wake_any_sem(sem); - spin_unlock(&sem->lock); + if (refcount_read(&sem->all_hint) > 1) { + spin_lock(&dev->wait_all_lock); + spin_lock(&sem->lock); + + prev_count = sem->u.sem.count; + ret = put_sem_state(sem, args.count); + if (!ret) { + try_wake_all_obj(dev, sem); + try_wake_any_sem(sem); + } + + spin_unlock(&sem->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&sem->lock); + + prev_count = sem->u.sem.count; + ret = put_sem_state(sem, args.count); + if (!ret) + try_wake_any_sem(sem); + + spin_unlock(&sem->lock); + } put_obj(sem); @@ -278,7 +413,7 @@ static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) * yet. Also, calculate the relative timeout in jiffies. */ static int setup_wait(struct winesync_device *dev, - const struct winesync_wait_args *args, + const struct winesync_wait_args *args, bool all, ktime_t *ret_timeout, struct winesync_q **ret_q) { const __u32 count = args->count; @@ -315,6 +450,7 @@ static int setup_wait(struct winesync_device *dev, q->task = current; q->owner = args->owner; atomic_set(&q->signaled, -1); + q->all = all; q->count = count; for (i = 0; i < count; i++) { @@ -324,6 +460,16 @@ static int setup_wait(struct winesync_device *dev, if (!obj) goto err; + if (all) { + /* Check that the objects are all distinct. */ + for (j = 0; j < i; j++) { + if (obj == q->entries[j].obj) { + put_obj(obj); + goto err; + } + } + } + entry->obj = obj; entry->q = q; entry->index = i; @@ -365,7 +511,7 @@ static int winesync_wait_any(struct winesync_device *dev, void __user *argp) if (!args.owner) return -EINVAL; - ret = setup_wait(dev, &args, &timeout, &q); + ret = setup_wait(dev, &args, false, &timeout, &q); if (ret < 0) return ret; @@ -425,6 +571,82 @@ static int winesync_wait_any(struct winesync_device *dev, void __user *argp) return ret; } +static int winesync_wait_all(struct winesync_device *dev, void __user *argp) +{ + struct winesync_wait_args args; + struct winesync_q *q; + ktime_t timeout; + __u32 i; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (!args.owner) + return -EINVAL; + + ret = setup_wait(dev, &args, true, &timeout, &q); + if (ret < 0) + return ret; + + /* queue ourselves */ + + spin_lock(&dev->wait_all_lock); + + for (i = 0; i < args.count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = q->entries[i].obj; + + refcount_inc(&obj->all_hint); + + /* + * obj->all_waiters is protected by dev->wait_all_lock rather + * than obj->lock, so there is no need to acquire it here. + */ + list_add_tail(&entry->node, &obj->all_waiters); + } + + /* check if we are already signaled */ + + try_wake_all(dev, q, NULL); + + spin_unlock(&dev->wait_all_lock); + + /* sleep */ + + ret = winesync_schedule(q, args.timeout ? &timeout : NULL); + + /* and finally, unqueue */ + + spin_lock(&dev->wait_all_lock); + + for (i = 0; i < args.count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = q->entries[i].obj; + + /* + * obj->all_waiters is protected by dev->wait_all_lock rather + * than obj->lock, so there is no need to acquire it here. + */ + list_del(&entry->node); + + refcount_dec(&obj->all_hint); + + put_obj(obj); + } + + spin_unlock(&dev->wait_all_lock); + + if (atomic_read(&q->signaled) != -1) { + /* even if we caught a signal, we need to communicate success */ + ret = 0; + } else if (!ret) { + ret = -ETIMEDOUT; + } + + kfree(q); + return ret; +} + static long winesync_char_ioctl(struct file *file, unsigned int cmd, unsigned long parm) { @@ -440,6 +662,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_put_sem(dev, argp); case WINESYNC_IOC_WAIT_ANY: return winesync_wait_any(dev, argp); + case WINESYNC_IOC_WAIT_ALL: + return winesync_wait_all(dev, argp); default: return -ENOSYS; } diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index c9e18310e77f0a..f4d77629486815 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -34,5 +34,7 @@ struct winesync_wait_args { #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ struct winesync_wait_args) +#define WINESYNC_IOC_WAIT_ALL _IOW (WINESYNC_IOC_BASE, 12, \ + struct winesync_wait_args) #endif From ead3d5c6feeb92cb59a1525de013498a7594c7d1 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:41:10 -0600 Subject: [PATCH 27/48] winesync: Introduce WINESYNC_IOC_CREATE_MUTEX. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 74 +++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 8 ++++ 2 files changed, 82 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index fe87082f80a728..9adf3d05bc5709 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -17,6 +17,7 @@ enum winesync_type { WINESYNC_TYPE_SEM, + WINESYNC_TYPE_MUTEX, }; struct winesync_obj { @@ -62,6 +63,10 @@ struct winesync_obj { __u32 count; __u32 max; } sem; + struct { + __u32 count; + __u32 owner; + } mutex; } u; }; @@ -182,6 +187,10 @@ static bool is_signaled(struct winesync_obj *obj, __u32 owner) switch (obj->type) { case WINESYNC_TYPE_SEM: return !!obj->u.sem.count; + case WINESYNC_TYPE_MUTEX: + if (obj->u.mutex.owner && obj->u.mutex.owner != owner) + return false; + return obj->u.mutex.count < UINT_MAX; } WARN(1, "bad object type %#x\n", obj->type); @@ -224,6 +233,10 @@ static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, case WINESYNC_TYPE_SEM: obj->u.sem.count--; break; + case WINESYNC_TYPE_MUTEX: + obj->u.mutex.count++; + obj->u.mutex.owner = q->owner; + break; } } wake_up_process(q->task); @@ -266,6 +279,28 @@ static void try_wake_any_sem(struct winesync_obj *sem) } } +static void try_wake_any_mutex(struct winesync_obj *mutex) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&mutex->lock); + + list_for_each_entry(entry, &mutex->any_waiters, node) { + struct winesync_q *q = entry->q; + + if (mutex->u.mutex.count == UINT_MAX) + break; + if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) + continue; + + if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + mutex->u.mutex.count++; + mutex->u.mutex.owner = q->owner; + wake_up_process(q->task); + } + } +} + static int winesync_create_sem(struct winesync_device *dev, void __user *argp) { struct winesync_sem_args __user *user_args = argp; @@ -300,6 +335,40 @@ static int winesync_create_sem(struct winesync_device *dev, void __user *argp) return put_user(ret, &user_args->sem); } +static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (!args.owner != !args.count) + return -EINVAL; + + mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); + if (!mutex) + return -ENOMEM; + + winesync_init_obj(mutex); + mutex->type = WINESYNC_TYPE_MUTEX; + mutex->u.mutex.count = args.count; + mutex->u.mutex.owner = args.owner; + + spin_lock(&dev->table_lock); + ret = idr_alloc(&dev->objects, mutex, 0, 0, GFP_KERNEL); + spin_unlock(&dev->table_lock); + + if (ret < 0) { + kfree(mutex); + return ret; + } + + return put_user(ret, &user_args->mutex); +} + static int winesync_delete(struct winesync_device *dev, void __user *argp) { struct winesync_obj *obj; @@ -495,6 +564,9 @@ static void try_wake_any_obj(struct winesync_obj *obj) case WINESYNC_TYPE_SEM: try_wake_any_sem(obj); break; + case WINESYNC_TYPE_MUTEX: + try_wake_any_mutex(obj); + break; } } @@ -656,6 +728,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, switch (cmd) { case WINESYNC_IOC_CREATE_SEM: return winesync_create_sem(dev, argp); + case WINESYNC_IOC_CREATE_MUTEX: + return winesync_create_mutex(dev, argp); case WINESYNC_IOC_DELETE: return winesync_delete(dev, argp); case WINESYNC_IOC_PUT_SEM: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index f4d77629486815..2e16652efe0356 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -16,6 +16,12 @@ struct winesync_sem_args { __u32 max; }; +struct winesync_mutex_args { + __u32 mutex; + __u32 owner; + __u32 count; +}; + struct winesync_wait_args { __u64 timeout; __u64 objs; @@ -29,6 +35,8 @@ struct winesync_wait_args { #define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ struct winesync_sem_args) +#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 1, \ + struct winesync_mutex_args) #define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ struct winesync_sem_args) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) From 13726fdedb27aff6e185a72b697ccddd5d4acc62 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:44:41 -0600 Subject: [PATCH 28/48] winesync: Introduce WINESYNC_IOC_PUT_MUTEX. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 71 +++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 2 + 2 files changed, 73 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 9adf3d05bc5709..0820ce8e8fb8b5 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -455,6 +455,75 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) return ret; } +/* + * Actually change the mutex state, returning -EPERM if not the owner. + */ +static int put_mutex_state(struct winesync_obj *mutex, + const struct winesync_mutex_args *args) +{ + lockdep_assert_held(&mutex->lock); + + if (mutex->u.mutex.owner != args->owner) + return -EPERM; + + if (!--mutex->u.mutex.count) + mutex->u.mutex.owner = 0; + return 0; +} + +static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + __u32 prev_count; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (!args.owner) + return -EINVAL; + + mutex = get_obj(dev, args.mutex); + if (!mutex) + return -EINVAL; + if (mutex->type != WINESYNC_TYPE_MUTEX) { + put_obj(mutex); + return -EINVAL; + } + + if (refcount_read(&mutex->all_hint) > 1) { + spin_lock(&dev->wait_all_lock); + spin_lock(&mutex->lock); + + prev_count = mutex->u.mutex.count; + ret = put_mutex_state(mutex, &args); + if (!ret) { + try_wake_all_obj(dev, mutex); + try_wake_any_mutex(mutex); + } + + spin_unlock(&mutex->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&mutex->lock); + + prev_count = mutex->u.mutex.count; + ret = put_mutex_state(mutex, &args); + if (!ret) + try_wake_any_mutex(mutex); + + spin_unlock(&mutex->lock); + } + + put_obj(mutex); + + if (!ret && put_user(prev_count, &user_args->count)) + ret = -EFAULT; + + return ret; +} + static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) { int ret = 0; @@ -734,6 +803,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_delete(dev, argp); case WINESYNC_IOC_PUT_SEM: return winesync_put_sem(dev, argp); + case WINESYNC_IOC_PUT_MUTEX: + return winesync_put_mutex(dev, argp); case WINESYNC_IOC_WAIT_ANY: return winesync_wait_any(dev, argp); case WINESYNC_IOC_WAIT_ALL: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 2e16652efe0356..2c8658d5dbe0df 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -39,6 +39,8 @@ struct winesync_wait_args { struct winesync_mutex_args) #define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ struct winesync_sem_args) +#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ + struct winesync_mutex_args) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ struct winesync_wait_args) From bff9174cee8ed35c1f74cae82e24f78790e1605f Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:46:46 -0600 Subject: [PATCH 29/48] winesync: Introduce WINESYNC_IOC_KILL_OWNER. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 84 +++++++++++++++++++++++++++++++++-- include/uapi/linux/winesync.h | 1 + 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 0820ce8e8fb8b5..b3d5c0bce150ee 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -66,6 +66,7 @@ struct winesync_obj { struct { __u32 count; __u32 owner; + bool ownerdead; } mutex; } u; }; @@ -89,6 +90,7 @@ struct winesync_q { atomic_t signaled; bool all; + bool ownerdead; __u32 count; struct winesync_q_entry entries[]; }; @@ -234,6 +236,9 @@ static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, obj->u.sem.count--; break; case WINESYNC_TYPE_MUTEX: + if (obj->u.mutex.ownerdead) + q->ownerdead = true; + obj->u.mutex.ownerdead = false; obj->u.mutex.count++; obj->u.mutex.owner = q->owner; break; @@ -294,6 +299,9 @@ static void try_wake_any_mutex(struct winesync_obj *mutex) continue; if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + if (mutex->u.mutex.ownerdead) + q->ownerdead = true; + mutex->u.mutex.ownerdead = false; mutex->u.mutex.count++; mutex->u.mutex.owner = q->owner; wake_up_process(q->task); @@ -357,9 +365,9 @@ static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) mutex->u.mutex.count = args.count; mutex->u.mutex.owner = args.owner; - spin_lock(&dev->table_lock); + mutex_lock(&dev->table_lock); ret = idr_alloc(&dev->objects, mutex, 0, 0, GFP_KERNEL); - spin_unlock(&dev->table_lock); + mutex_unlock(&dev->table_lock); if (ret < 0) { kfree(mutex); @@ -524,6 +532,71 @@ static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) return ret; } +/* + * Actually change the mutex state to mark its owner as dead. + */ +static void put_mutex_ownerdead_state(struct winesync_obj *mutex) +{ + lockdep_assert_held(&mutex->lock); + + mutex->u.mutex.ownerdead = true; + mutex->u.mutex.owner = 0; + mutex->u.mutex.count = 0; +} + +static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) +{ + struct winesync_obj *obj; + __u32 owner; + int id; + + if (get_user(owner, (__u32 __user *)argp)) + return -EFAULT; + if (!owner) + return -EINVAL; + + rcu_read_lock(); + + idr_for_each_entry(&dev->objects, obj, id) { + if (!kref_get_unless_zero(&obj->refcount)) + continue; + + if (obj->type != WINESYNC_TYPE_MUTEX) { + put_obj(obj); + continue; + } + + if (refcount_read(&obj->all_hint) > 1) { + spin_lock(&dev->wait_all_lock); + spin_lock(&obj->lock); + + if (obj->u.mutex.owner == owner) { + put_mutex_ownerdead_state(obj); + try_wake_all_obj(dev, obj); + try_wake_any_mutex(obj); + } + + spin_unlock(&obj->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&obj->lock); + + if (obj->u.mutex.owner == owner) { + put_mutex_ownerdead_state(obj); + try_wake_any_mutex(obj); + } + + spin_unlock(&obj->lock); + } + + put_obj(obj); + } + + rcu_read_unlock(); + + return 0; +} + static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) { int ret = 0; @@ -589,6 +662,7 @@ static int setup_wait(struct winesync_device *dev, q->owner = args->owner; atomic_set(&q->signaled, -1); q->all = all; + q->ownerdead = false; q->count = count; for (i = 0; i < count; i++) { @@ -700,7 +774,7 @@ static int winesync_wait_any(struct winesync_device *dev, void __user *argp) struct winesync_wait_args __user *user_args = argp; /* even if we caught a signal, we need to communicate success */ - ret = 0; + ret = q->ownerdead ? -EOWNERDEAD : 0; if (put_user(atomic_read(&q->signaled), &user_args->index)) ret = -EFAULT; @@ -779,7 +853,7 @@ static int winesync_wait_all(struct winesync_device *dev, void __user *argp) if (atomic_read(&q->signaled) != -1) { /* even if we caught a signal, we need to communicate success */ - ret = 0; + ret = q->ownerdead ? -EOWNERDEAD : 0; } else if (!ret) { ret = -ETIMEDOUT; } @@ -805,6 +879,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_put_sem(dev, argp); case WINESYNC_IOC_PUT_MUTEX: return winesync_put_mutex(dev, argp); + case WINESYNC_IOC_KILL_OWNER: + return winesync_kill_owner(dev, argp); case WINESYNC_IOC_WAIT_ANY: return winesync_wait_any(dev, argp); case WINESYNC_IOC_WAIT_ALL: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 2c8658d5dbe0df..d50d76d66705d9 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -41,6 +41,7 @@ struct winesync_wait_args { struct winesync_sem_args) #define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ struct winesync_mutex_args) +#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 9, __u32) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ struct winesync_wait_args) From 0f71ea560c57873eb4e3c63b85afbeeb66c70d1b Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:47:55 -0600 Subject: [PATCH 30/48] winesync: Introduce WINESYNC_IOC_READ_SEM. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 33 +++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 2 ++ 2 files changed, 35 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index b3d5c0bce150ee..9d0d1a42ebc827 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -532,6 +532,37 @@ static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) return ret; } +static int winesync_read_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + __u32 id; + + if (get_user(id, &user_args->sem)) + return -EFAULT; + + sem = get_obj(dev, id); + if (!sem) + return -EINVAL; + if (sem->type != WINESYNC_TYPE_SEM) { + put_obj(sem); + return -EINVAL; + } + + args.sem = id; + spin_lock(&sem->lock); + args.count = sem->u.sem.count; + args.max = sem->u.sem.max; + spin_unlock(&sem->lock); + + put_obj(sem); + + if (copy_to_user(user_args, &args, sizeof(args))) + return -EFAULT; + return 0; +} + /* * Actually change the mutex state to mark its owner as dead. */ @@ -879,6 +910,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_put_sem(dev, argp); case WINESYNC_IOC_PUT_MUTEX: return winesync_put_mutex(dev, argp); + case WINESYNC_IOC_READ_SEM: + return winesync_read_sem(dev, argp); case WINESYNC_IOC_KILL_OWNER: return winesync_kill_owner(dev, argp); case WINESYNC_IOC_WAIT_ANY: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index d50d76d66705d9..15d770683b1a93 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -41,6 +41,8 @@ struct winesync_wait_args { struct winesync_sem_args) #define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ struct winesync_mutex_args) +#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 7, \ + struct winesync_sem_args) #define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 9, __u32) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ From cd26c144d1caef5dc5ce38e60f5c7b60a0a91229 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:48:10 -0600 Subject: [PATCH 31/48] winesync: Introduce WINESYNC_IOC_READ_MUTEX. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 35 +++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 2 ++ 2 files changed, 37 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 9d0d1a42ebc827..c28a45c979df49 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -563,6 +563,39 @@ static int winesync_read_sem(struct winesync_device *dev, void __user *argp) return 0; } +static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + __u32 id; + int ret; + + if (get_user(id, &user_args->mutex)) + return -EFAULT; + + mutex = get_obj(dev, id); + if (!mutex) + return -EINVAL; + if (mutex->type != WINESYNC_TYPE_MUTEX) { + put_obj(mutex); + return -EINVAL; + } + + args.mutex = id; + spin_lock(&mutex->lock); + args.count = mutex->u.mutex.count; + args.owner = mutex->u.mutex.owner; + ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; + spin_unlock(&mutex->lock); + + put_obj(mutex); + + if (copy_to_user(user_args, &args, sizeof(args))) + return -EFAULT; + return ret; +} + /* * Actually change the mutex state to mark its owner as dead. */ @@ -912,6 +945,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_put_mutex(dev, argp); case WINESYNC_IOC_READ_SEM: return winesync_read_sem(dev, argp); + case WINESYNC_IOC_READ_MUTEX: + return winesync_read_mutex(dev, argp); case WINESYNC_IOC_KILL_OWNER: return winesync_kill_owner(dev, argp); case WINESYNC_IOC_WAIT_ANY: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 15d770683b1a93..1fdc8801845ff2 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -43,6 +43,8 @@ struct winesync_wait_args { struct winesync_mutex_args) #define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 7, \ struct winesync_sem_args) +#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 8, \ + struct winesync_mutex_args) #define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 9, __u32) #define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 10, __u32) #define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 11, \ From 9404ca99dddafe7ea842af6cfbd5b0c482dad862 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 11:50:49 -0600 Subject: [PATCH 32/48] doc: Add documentation for the winesync uAPI. Signed-off-by: Kai Krakow --- Documentation/userspace-api/index.rst | 1 + Documentation/userspace-api/winesync.rst | 315 +++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 Documentation/userspace-api/winesync.rst diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index acd2cc2a538df5..1950883c333e1e 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -24,6 +24,7 @@ place where this information is gathered. ioctl/index iommu media/index + winesync .. only:: subproject and html diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst new file mode 100644 index 00000000000000..7b4e5767c3ece4 --- /dev/null +++ b/Documentation/userspace-api/winesync.rst @@ -0,0 +1,315 @@ +===================================== +Wine synchronization primitive driver +===================================== + +This page documents the user-space API for the winesync driver. + +winesync is a support driver for emulation of NT synchronization +primitives by the Wine project. It exists because implementation in +user-space, using existing tools, cannot satisfy performance, +correctness, and security constraints. It is implemented entirely in +software, and does not drive any hardware device. + +This interface is meant as a compatibility tool only and should not be +used for general synchronization; instead use generic, versatile +interfaces such as futex(2) and poll(2). + +Synchronization primitives +========================== + +The winesync driver exposes two types of synchronization primitives, +semaphores and mutexes. + +A semaphore holds a single volatile 32-bit counter, and a static +32-bit integer denoting the maximum value. It is considered signaled +when the counter is nonzero. The counter is decremented by one when a +wait is satisfied. Both the initial and maximum count are established +when the semaphore is created. + +A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit +identifier denoting its owner. The latter is intended to identify the +thread holding the mutex; however, it is not actually validated +against earlier calls made by the same thread. A mutex is considered +signaled when its owner is zero (indicating that it is not owned). The +recursion count is incremented when a wait is satisfied, and ownership +is set to the given identifier. A mutex also holds an internal flag +denoting whether its previous owner has died; such a mutex is said to +be inconsistent. Owner death is not tracked automatically based on +thread death, but rather must be communicated using +``WINESYNC_IOC_KILL_OWNER``. + +Objects are represented by unsigned 32-bit integers. An object +identifier will always be less than or equal to the value of INT_MAX. + +Char device +=========== + +The winesync driver creates a single char device /dev/winesync. Each +file description opened on the device represents a unique namespace. +That is, objects created on one open file description are shared +across all its individual descriptors, but are not shared with other +open() calls on the same device. + +ioctl reference +=============== + +All operations on the device are done through ioctls. There are three +structures used in ioctl calls:: + + struct winesync_sem_args { + __u32 sem; + __u32 count; + __u32 max; + }; + + struct winesync_mutex_args { + __u32 mutex; + __u32 owner; + __u32 count; + }; + + struct winesync_wait_args { + __u64 timeout; + __u64 objs; + __u32 count; + __u32 owner; + __u32 index; + __u32 pad; + }; + +Depending on the ioctl, members of the structure may be used as input, +output, or not at all. + +All ioctls return 0 on success, and -1 on error, in which case `errno` +will be set to a nonzero error code. + +The ioctls are as follows: + +.. c:macro:: WINESYNC_IOC_CREATE_SEM + + Create a semaphore object. Takes a pointer to struct + :c:type:`winesync_sem_args`, which is used as follows: + + ``count`` and ``max`` are input-only arguments, denoting the + initial and maximum count of the semaphore. + + ``sem`` is an output-only argument, which will be filled with the + allocated identifier if successful. + + Fails with ``EINVAL`` if ``count`` is greater than ``max``, or + ``ENOMEM`` if not enough memory is available. + +.. c:macro:: WINESYNC_IOC_CREATE_MUTEX + + Create a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + ``owner`` is an input-only argument denoting the initial owner of + the mutex. + + ``count`` is an input-only argument denoting the initial recursion + count of the mutex. If ``owner`` is nonzero and ``count`` is zero, + or if ``owner`` is zero and ``count`` is nonzero, the function + fails with ``EINVAL``. + + ``mutex`` is an output-only argument, which will be filled with + the allocated identifier if successful. + + Fails with ``ENOMEM`` if not enough memory is available. + +.. c:macro:: WINESYNC_IOC_DELETE + + Delete an object of any type. Takes an input-only pointer to a + 32-bit integer denoting the object to delete. Fails with ``EINVAL`` + if the object is not valid. Further ioctls attempting to use the + object return ``EINVAL``, unless the object identifier is reused. + However, wait ioctls currently in progress are not interrupted, and + behave as if the object remains valid. + +.. c:macro:: WINESYNC_IOC_PUT_SEM + + Post to a semaphore object. Takes a pointer to struct + :c:type:`winesync_sem_args`, which is used as follows: + + ``sem`` is an input-only argument denoting the semaphore object. + If ``sem`` is not a valid semaphore object, the ioctl fails with + ``EINVAL``. + + ``count`` contains on input the count to add to the semaphore, and + on output is filled with its previous count. + + ``max`` is not used. + + The operation is atomic and totally ordered with respect to other + operations on the same semaphore. If adding ``count`` to the + semaphore's current count would raise the latter past the + semaphore's maximum count, the ioctl fails with ``EOVERFLOW`` and + the semaphore is not affected. If raising the semaphore's count + causes it to become signaled, eligible threads waiting on this + semaphore will be woken and the semaphore's count decremented + appropriately. + +.. c:macro:: WINESYNC_IOC_PUT_MUTEX + + Release a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + ``mutex`` is an input-only argument denoting the mutex object. If + ``mutex`` is not a valid mutex object, the ioctl fails with + ``EINVAL``. + + ``owner`` is an input-only argument denoting the mutex owner. + ``owner`` must be nonzero, else the ioctl fails with ``EINVAL``. + If ``owner`` is not the current owner of the mutex, the ioctl + fails with ``EPERM``. + + ``count`` is an output-only argument which will be filled on + success with the mutex's previous recursion count. + + The mutex's count will be decremented by one. The operation is + atomic and totally ordered with respect to other operations on the + same mutex. If decrementing the mutex's count causes it to become + zero, the mutex is marked as unowned and signaled, and eligible + threads waiting on it will be woken as appropriate. + +.. c:macro:: WINESYNC_IOC_READ_SEM + + Read the current state of a semaphore object. Takes a pointer to + struct :c:type:`winesync_sem_args`, which is used as follows: + + ``sem`` is an input-only argument denoting the semaphore object. + If ``sem`` is not a valid semaphore object, the ioctl fails with + ``EINVAL``. + + ``count`` and ``max`` are output-only arguments, which will be + filled with the current and maximum count of the given semaphore. + + The operation is atomic and totally ordered with respect to other + operations on the same semaphore. + +.. c:macro:: WINESYNC_IOC_READ_MUTEX + + Read the current state of a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + ``mutex`` is an input-only argument denoting the mutex object. If + ``mutex`` is not a valid mutex object, the ioctl fails with + ``EINVAL``. + + ``count`` and ``owner`` are output-only arguments, which will be + filled with the current recursion count and owner of the given + mutex. If the mutex is not owned, both ``count`` and ``owner`` are + set to zero. + + If the mutex is marked as inconsistent, the function fails with + ``EOWNERDEAD``. + + The operation is atomic and totally ordered with respect to other + operations on the same mutex. + +.. c:macro:: WINESYNC_IOC_KILL_OWNER + + Mark any mutexes owned by the given identifier as unowned and + inconsistent. Takes an input-only pointer to a 32-bit integer + denoting the owner. If the owner is zero, the ioctl fails with + ``EINVAL``. + +.. c:macro:: WINESYNC_IOC_WAIT_ANY + + Poll on any of a list of objects, atomically acquiring (at most) + one. Takes a pointer to struct :c:type:`winesync_wait_args`, which + is used as follows: + + ``timeout`` is an optional input-only pointer to a 64-bit struct + :c:type:`timespec` (specified as an integer so that the structure + has the same size regardless of architecture). The timeout is + specified in absolute format, as measured against the MONOTONIC + clock. If the timeout is equal to or earlier than the current + time, the function returns immediately without sleeping. If + ``timeout`` is zero, i.e. NULL, the function will sleep until an + object is signaled, and will not fail with ``ETIMEDOUT``. + + ``objs`` is a input-only pointer to an array of ``count`` 32-bit + object identifiers (specified as an integer so that the structure + has the same size regardless of architecture). If any identifier + is invalid, the function fails with ``EINVAL``. + + ``count`` is an input-only argument denoting the number of + elements in ``objs``. + + ``owner`` is an input-only argument denoting the mutex owner + identifier. If any object in ``objs`` is a mutex, the ioctl will + attempt to acquire that mutex on behalf of ``owner``. If ``owner`` + is zero, the ioctl fails with ``EINVAL``. + + ``index`` is an output-only argument which, if the ioctl is + successful, is filled with the index of the object actually + signaled. + + ``pad`` is unused, and exists to keep a consistent structure size. + + This function attempts to acquire one of the given objects. If + unable to do so, it sleeps until an object becomes signaled, + subsequently acquiring it, or the timeout expires. In the latter + case the ioctl fails with ``ETIMEDOUT``. The function only acquires + one object, even if multiple objects are signaled. + + A semaphore is considered to be signaled if its count is nonzero, + and is acquired by decrementing its count by one. A mutex is + considered to be signaled if it is unowned or if its owner matches + the ``owner`` argument, and is acquired by incrementing its + recursion count by one and setting its owner to the ``owner`` + argument. + + Acquisition is atomic and totally ordered with respect to other + operations on the same object. If two wait operations (with + different ``owner`` identifiers) are queued on the same mutex, only + one is signaled. If two wait operations are queued on the same + semaphore, and a value of one is posted to it, only one is signaled. + The order in which threads are signaled is not guaranteed. + + If an inconsistent mutex is acquired, the ioctl fails with + ``EOWNERDEAD``. Although this is a failure return, the function may + otherwise be considered successful. The mutex is marked as owned by + the given owner (with a recursion count of 1) and as no longer + inconsistent. ``index`` is still set to the index of the mutex. + + Unlike ``WINESYNC_IOC_WAIT_ALL``, it is valid to pass the same + object more than once. If a wakeup occurs due to that object being + signaled, ``index`` is set to the index of the first instance of the + object. + + Fails with ``ENOMEM`` if not enough memory is available, or + ``EINTR`` if a signal is received. + +.. c:macro:: WINESYNC_IOC_WAIT_ALL + + Poll on a list of objects, atomically acquiring all of them. Takes a + pointer to struct :c:type:`winesync_wait_args`, which is used + identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is + unused. + + This function attempts to simultaneously acquire all of the given + objects. If unable to do so, it sleeps until all objects become + simultaneously signaled, subsequently acquiring them, or the timeout + expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and + no objects are modified. + + Objects may become signaled and subsequently designaled (through + acquisition by other threads) while this thread is sleeping. Only + once all objects are simultaneously signaled does the ioctl return. + The acquisition is atomic and totally ordered with respect to other + operations on any of the given objects. + + If an inconsistent mutex is acquired, the ioctl fails with + ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects + are nevertheless marked as acquired. Note that if multiple mutex + objects are specified, there is no way to know which were marked as + inconsistent. + + Unlike ``WINESYNC_IOC_WAIT_ALL``, it is not valid to pass the same + object more than once. If this is attempted, the function fails with + ``EINVAL``. + + Fails with ``ENOMEM`` if not enough memory is available, or + ``EINTR`` if a signal is received. From 10283c1e4495f699b514032e42e8c15ee902cb7b Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:06:23 -0600 Subject: [PATCH 33/48] selftests: winesync: Add some tests for semaphore state. Signed-off-by: Kai Krakow --- tools/testing/selftests/Makefile | 1 + .../selftests/drivers/winesync/Makefile | 8 + .../testing/selftests/drivers/winesync/config | 1 + .../selftests/drivers/winesync/winesync.c | 153 ++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 tools/testing/selftests/drivers/winesync/Makefile create mode 100644 tools/testing/selftests/drivers/winesync/config create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index d9c2835031590e..775fe4c0f65f4c 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -10,6 +10,7 @@ TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug TARGETS += drivers/dma-buf +TARGETS += drivers/winesync TARGETS += efivarfs TARGETS += exec TARGETS += filesystems diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile new file mode 100644 index 00000000000000..43b39fdeea10ec --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/Makefile @@ -0,0 +1,8 @@ +# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only +TEST_GEN_PROGS := winesync + +top_srcdir =../../../../.. +CFLAGS += -I$(top_srcdir)/usr/include +LDLIBS += -lpthread + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config new file mode 100644 index 00000000000000..60539c826d0624 --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/config @@ -0,0 +1 @@ +CONFIG_WINESYNC=y diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c new file mode 100644 index 00000000000000..077d9322923c38 --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Various unit tests for the "winesync" synchronization primitive driver. + * + * Copyright (C) 2021 Zebediah Figura + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../kselftest_harness.h" + +TEST(semaphore_state) +{ + struct winesync_wait_args wait_args; + struct winesync_sem_args sem_args; + struct timespec timeout; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 3; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + sem_args.count = 2; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 0; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)&sem_args.sem; + wait_args.count = 1; + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + sem_args.count = 3; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST_HARNESS_MAIN From e5211474482c84de6bc1a9c4c37640c5ac9c5344 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:07:04 -0600 Subject: [PATCH 34/48] selftests: winesync: Add some tests for mutex state. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 077d9322923c38..713711dd4bf030 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -150,4 +150,254 @@ TEST(semaphore_state) close(fd); } +TEST(mutex_state) +{ + struct winesync_wait_args wait_args; + struct winesync_mutex_args mutex_args; + struct timespec timeout; + __u32 owner; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + mutex_args.owner = 123; + mutex_args.count = 0; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + mutex_args.owner = 0; + mutex_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + mutex_args.owner = 123; + mutex_args.count = 2; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 456; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EPERM, errno); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EPERM, errno); + + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)&mutex_args.mutex; + wait_args.count = 1; + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 456; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + owner = 0; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + owner = 456; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, wait_args.index); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, wait_args.index); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + TEST_HARNESS_MAIN From 4d2f875da4938fce46c6cfc54695d0818dd629c6 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:07:45 -0600 Subject: [PATCH 35/48] selftests: winesync: Add some tests for WINESYNC_IOC_WAIT_ANY. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 713711dd4bf030..774e7a0b0efec9 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -400,4 +400,201 @@ TEST(mutex_state) close(fd); } +TEST(wait_any) +{ + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct timespec timeout; + __u32 objs[2], owner; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 2; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + EXPECT_EQ((uintptr_t)objs, wait_args.objs); + EXPECT_EQ(2, wait_args.count); + EXPECT_EQ(123, wait_args.owner); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(1, wait_args.index); + + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, wait_args.index); + + /* test waiting on the same object twice */ + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + objs[0] = objs[1] = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + wait_args.count = 0; + wait_args.objs = (uintptr_t)NULL; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + TEST_HARNESS_MAIN From 652a828231f9188e592cd2108dacb752a9ce5c57 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:08:25 -0600 Subject: [PATCH 36/48] selftests: winesync: Add some tests for WINESYNC_IOC_WAIT_ALL. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 774e7a0b0efec9..6c1f7fa325fd53 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -597,4 +597,155 @@ TEST(wait_any) close(fd); } +TEST(wait_all) +{ + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct timespec timeout; + __u32 objs[2], owner; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 2; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ((uintptr_t)objs, wait_args.objs); + EXPECT_EQ(2, wait_args.count); + EXPECT_EQ(123, wait_args.owner); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + wait_args.owner = 456; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + wait_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(0, ret); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + sem_args.count = 3; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(3, sem_args.max); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(123, mutex_args.owner); + + /* test waiting on the same object twice */ + objs[0] = objs[1] = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + TEST_HARNESS_MAIN From 0a4a6f6529dbd5008e9dbf68134e1e51708cd52b Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:08:54 -0600 Subject: [PATCH 37/48] selftests: winesync: Add some tests for invalid object handling. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 6c1f7fa325fd53..28e6d13afe7329 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -748,4 +748,97 @@ TEST(wait_all) close(fd); } +TEST(invalid_objects) +{ + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + __u32 objs[2] = {0}; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + wait_args.objs = (uintptr_t)objs; + wait_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + sem_args.max = 1; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + + mutex_args.mutex = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + objs[0] = sem_args.sem; + objs[1] = sem_args.sem + 1; + wait_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + objs[0] = sem_args.sem + 1; + objs[1] = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + + sem_args.sem = mutex_args.mutex; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + TEST_HARNESS_MAIN From 475278626e31fa2a8529612feca312727d8c648a Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:09:32 -0600 Subject: [PATCH 38/48] selftests: winesync: Add some tests for wakeup signaling with WINESYNC_IOC_WAIT_ANY. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 28e6d13afe7329..722a03c40645ae 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -841,4 +841,170 @@ TEST(invalid_objects) close(fd); } +struct wake_args +{ + int fd; + __u32 obj; +}; + +struct wait_args +{ + int fd; + unsigned long request; + struct winesync_wait_args *args; + int ret; + int err; +}; + +static void *wait_thread(void *arg) +{ + struct wait_args *args = arg; + + args->ret = ioctl(args->fd, args->request, args->args); + args->err = errno; + return NULL; +} + +static void get_abs_timeout(struct timespec *timeout, clockid_t clock, + unsigned int ms) +{ + clock_gettime(clock, timeout); + timeout->tv_nsec += ms * 1000000; + timeout->tv_sec += (timeout->tv_nsec / 1000000000); + timeout->tv_nsec %= 1000000000; +} + +static int wait_for_thread(pthread_t thread, unsigned int ms) +{ + struct timespec timeout; + get_abs_timeout(&timeout, CLOCK_REALTIME, ms); + return pthread_timedjoin_np(thread, NULL, &timeout); +} + +TEST(wake_any) +{ + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct wait_args thread_args; + struct timespec timeout; + __u32 objs[2], owner; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 0; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 123; + mutex_args.count = 1; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + /* test waking the semaphore */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ANY; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(0, wait_args.index); + + /* test waking the mutex */ + + /* first grab it again for owner 123 */ + wait_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, wait_args.index); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.owner = 456; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + mutex_args.owner = 123; + mutex_args.count = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, mutex_args.count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + mutex_args.owner = 123; + mutex_args.count = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + /* delete an object while it's being waited on */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); + wait_args.owner = 123; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 200); + EXPECT_EQ(0, ret); + EXPECT_EQ(-1, thread_args.ret); + EXPECT_EQ(ETIMEDOUT, thread_args.err); + + close(fd); +} + TEST_HARNESS_MAIN From fecaecced205a41904dd45100c13f610bed8005c Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:09:36 -0600 Subject: [PATCH 39/48] selftests: winesync: Add some tests for wakeup signaling with WINESYNC_IOC_WAIT_ALL. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 722a03c40645ae..d4a2ed9f5f2213 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -1007,4 +1007,125 @@ TEST(wake_any) close(fd); } +TEST(wake_all) +{ + struct winesync_wait_args wait_args = {0}, wait_args2 = {0}; + struct winesync_mutex_args mutex_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct timespec timeout, timeout2; + struct wait_args thread_args; + __u32 objs[2], owner; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 0; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 123; + mutex_args.count = 1; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 456; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ALL; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + + get_abs_timeout(&timeout2, CLOCK_MONOTONIC, 0); + wait_args2.timeout = (uintptr_t)&timeout2; + wait_args2.objs = (uintptr_t)&sem_args.sem; + wait_args2.count = 1; + wait_args2.owner = 123; + wait_args2.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args2); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args2.index); + + mutex_args.owner = 123; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + EXPECT_EQ(456, mutex_args.owner); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + + /* delete an object while it's being waited on */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); + wait_args.owner = 123; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 200); + EXPECT_EQ(0, ret); + EXPECT_EQ(-1, thread_args.ret); + EXPECT_EQ(ETIMEDOUT, thread_args.err); + + close(fd); +} + TEST_HARNESS_MAIN From 078a1e8d785bd9d4e4c0fe1bf24fe07d9d815cfe Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 12:22:55 -0600 Subject: [PATCH 40/48] maintainers: Add an entry for winesync. Signed-off-by: Kai Krakow --- MAINTAINERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 1757eb0d65a180..7265c05679f3c4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18919,6 +18919,15 @@ M: David Härdeman S: Maintained F: drivers/media/rc/winbond-cir.c +WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER +M: Zebediah Figura +L: wine-devel@winehq.org +S: Supported +F: Documentation/userspace-api/winesync.rst +F: drivers/misc/winesync.c +F: include/uapi/linux/winesync.c +F: tools/testing/selftests/drivers/winesync/ + WINSYSTEMS EBC-C384 WATCHDOG DRIVER M: William Breathitt Gray L: linux-watchdog@vger.kernel.org From 787f6b8805f3f28dcd1630d740e26ffa32088682 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 16:12:55 -0600 Subject: [PATCH 41/48] winesync: Introduce the WINESYNC_SEM_GETONWAIT flag. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 12 ++++++++++-- include/uapi/linux/winesync.h | 3 +++ tools/testing/selftests/drivers/winesync/winesync.c | 5 +++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index c28a45c979df49..8de31c3b4448c4 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -62,6 +62,7 @@ struct winesync_obj { struct { __u32 count; __u32 max; + __u32 flags; } sem; struct { __u32 count; @@ -233,7 +234,8 @@ static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, switch (obj->type) { case WINESYNC_TYPE_SEM: - obj->u.sem.count--; + if (obj->u.sem.flags & WINESYNC_SEM_GETONWAIT) + obj->u.sem.count--; break; case WINESYNC_TYPE_MUTEX: if (obj->u.mutex.ownerdead) @@ -278,7 +280,8 @@ static void try_wake_any_sem(struct winesync_obj *sem) break; if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { - sem->u.sem.count--; + if (sem->u.sem.flags & WINESYNC_SEM_GETONWAIT) + sem->u.sem.count--; wake_up_process(q->task); } } @@ -322,6 +325,9 @@ static int winesync_create_sem(struct winesync_device *dev, void __user *argp) if (args.count > args.max) return -EINVAL; + if (args.flags & ~WINESYNC_SEM_GETONWAIT) + return -EINVAL; + sem = kzalloc(sizeof(*sem), GFP_KERNEL); if (!sem) return -ENOMEM; @@ -330,6 +336,7 @@ static int winesync_create_sem(struct winesync_device *dev, void __user *argp) sem->type = WINESYNC_TYPE_SEM; sem->u.sem.count = args.count; sem->u.sem.max = args.max; + sem->u.sem.flags = args.flags; mutex_lock(&dev->table_lock); ret = idr_alloc(&dev->objects, sem, 0, 0, GFP_KERNEL); @@ -554,6 +561,7 @@ static int winesync_read_sem(struct winesync_device *dev, void __user *argp) spin_lock(&sem->lock); args.count = sem->u.sem.count; args.max = sem->u.sem.max; + args.flags = sem->u.sem.flags; spin_unlock(&sem->lock); put_obj(sem); diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 1fdc8801845ff2..8acf54e997ba5e 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -10,10 +10,13 @@ #include +#define WINESYNC_SEM_GETONWAIT 1 + struct winesync_sem_args { __u32 sem; __u32 count; __u32 max; + __u32 flags; }; struct winesync_mutex_args { diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index d4a2ed9f5f2213..97719feb625e5a 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -29,6 +29,7 @@ TEST(semaphore_state) sem_args.count = 3; sem_args.max = 2; sem_args.sem = 0xdeadbeef; + sem_args.flags = WINESYNC_SEM_GETONWAIT; ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); EXPECT_EQ(-1, ret); EXPECT_EQ(EINVAL, errno); @@ -417,6 +418,7 @@ TEST(wait_any) sem_args.count = 2; sem_args.max = 3; sem_args.sem = 0xdeadbeef; + sem_args.flags = WINESYNC_SEM_GETONWAIT; ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); EXPECT_EQ(0, ret); EXPECT_NE(0xdeadbeef, sem_args.sem); @@ -614,6 +616,7 @@ TEST(wait_all) sem_args.count = 2; sem_args.max = 3; sem_args.sem = 0xdeadbeef; + sem_args.flags = WINESYNC_SEM_GETONWAIT; ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); EXPECT_EQ(0, ret); EXPECT_NE(0xdeadbeef, sem_args.sem); @@ -898,6 +901,7 @@ TEST(wake_any) sem_args.count = 0; sem_args.max = 3; sem_args.sem = 0xdeadbeef; + sem_args.flags = WINESYNC_SEM_GETONWAIT; ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); EXPECT_EQ(0, ret); EXPECT_NE(0xdeadbeef, sem_args.sem); @@ -1024,6 +1028,7 @@ TEST(wake_all) sem_args.count = 0; sem_args.max = 3; sem_args.sem = 0xdeadbeef; + sem_args.flags = WINESYNC_SEM_GETONWAIT; ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); EXPECT_EQ(0, ret); EXPECT_NE(0xdeadbeef, sem_args.sem); From 010e0198b426281ac93422b4eb67aa8f78a9f94b Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 16:13:07 -0600 Subject: [PATCH 42/48] doc: Document the WINESYNC_SEM_GETONWAIT flag. Signed-off-by: Kai Krakow --- Documentation/userspace-api/winesync.rst | 34 ++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst index 7b4e5767c3ece4..3c5e986ddd8c1f 100644 --- a/Documentation/userspace-api/winesync.rst +++ b/Documentation/userspace-api/winesync.rst @@ -60,6 +60,7 @@ structures used in ioctl calls:: __u32 sem; __u32 count; __u32 max; + __u32 flags; }; struct winesync_mutex_args { @@ -93,6 +94,13 @@ The ioctls are as follows: ``count`` and ``max`` are input-only arguments, denoting the initial and maximum count of the semaphore. + ``flags`` is an input-only argument, which specifies additional + flags modifying the behaviour of the semaphore. There is only one + flag defined, ``WINESYNC_SEM_GETONWAIT``. If present, wait + operations on this semaphore will acquire it, decrementing its + count by one; otherwise, wait operations will not affect the + semaphore's state. + ``sem`` is an output-only argument, which will be filled with the allocated identifier if successful. @@ -138,7 +146,7 @@ The ioctls are as follows: ``count`` contains on input the count to add to the semaphore, and on output is filled with its previous count. - ``max`` is not used. + ``max`` and ``flags`` are not used. The operation is atomic and totally ordered with respect to other operations on the same semaphore. If adding ``count`` to the @@ -184,6 +192,9 @@ The ioctls are as follows: ``count`` and ``max`` are output-only arguments, which will be filled with the current and maximum count of the given semaphore. + ``flags`` is an output-only argument, which will be filled with + the flags used to create the semaphore. + The operation is atomic and totally ordered with respect to other operations on the same semaphore. @@ -254,20 +265,27 @@ The ioctls are as follows: case the ioctl fails with ``ETIMEDOUT``. The function only acquires one object, even if multiple objects are signaled. - A semaphore is considered to be signaled if its count is nonzero, - and is acquired by decrementing its count by one. A mutex is - considered to be signaled if it is unowned or if its owner matches - the ``owner`` argument, and is acquired by incrementing its - recursion count by one and setting its owner to the ``owner`` - argument. + A semaphore is considered to be signaled if its count is nonzero. It + is acquired by decrementing its count by one if the + ``WINESYNC_SEM_GETONWAIT`` flag was used to create it; otherwise no + operation is done to acquire the semaphore. A mutex is considered to + be signaled if it is unowned or if its owner matches the ``owner`` + argument, and is acquired by incrementing its recursion count by one + and setting its owner to the ``owner`` argument. Acquisition is atomic and totally ordered with respect to other operations on the same object. If two wait operations (with different ``owner`` identifiers) are queued on the same mutex, only one is signaled. If two wait operations are queued on the same - semaphore, and a value of one is posted to it, only one is signaled. + semaphore (which was not created with the ``WINESYNC_SEM_GETONWAIT`` + flag set), and a value of one is posted to it, only one is signaled. The order in which threads are signaled is not guaranteed. + (If two wait operations are queued on the same semaphore, and the + semaphore was created with the ``WINESYNC_SEM_GETONWAIT`` flag set, + and a value of one is posted to it, both threads are signaled, and + the semaphore retains a count of one.) + If an inconsistent mutex is acquired, the ioctl fails with ``EOWNERDEAD``. Although this is a failure return, the function may otherwise be considered successful. The mutex is marked as owned by From 76e590a43419398754df8856894ee656c2aee656 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 16:19:53 -0600 Subject: [PATCH 43/48] winesync: Introduce WINESYNC_IOC_GET_SEM. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 37 +++++++++++++++++++++++++++++++++++ include/uapi/linux/winesync.h | 1 + 2 files changed, 38 insertions(+) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index 8de31c3b4448c4..d4779d09ab91d1 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -403,6 +403,41 @@ static int winesync_delete(struct winesync_device *dev, void __user *argp) return 0; } +static int winesync_get_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_obj *sem; + int ret = -EWOULDBLOCK; + __u32 id; + + if (get_user(id, (__u32 __user *)argp)) + return -EFAULT; + + sem = get_obj(dev, id); + if (!sem) + return -EINVAL; + if (sem->type != WINESYNC_TYPE_SEM) { + put_obj(sem); + return -EINVAL; + } + + spin_lock(&sem->lock); + + if (sem->u.sem.count) { + /* + * Decrement the semaphore's count, regardless of whether it + * has the WINESYNC_SEM_GETONWAIT flag set. + */ + sem->u.sem.count--; + ret = 0; + } + + spin_unlock(&sem->lock); + + put_obj(sem); + + return ret; +} + /* * Actually change the semaphore state, returning -EOVERFLOW if it is made * invalid. @@ -947,6 +982,8 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, return winesync_create_mutex(dev, argp); case WINESYNC_IOC_DELETE: return winesync_delete(dev, argp); + case WINESYNC_IOC_GET_SEM: + return winesync_get_sem(dev, argp); case WINESYNC_IOC_PUT_SEM: return winesync_put_sem(dev, argp); case WINESYNC_IOC_PUT_MUTEX: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 8acf54e997ba5e..96bceacb809126 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -40,6 +40,7 @@ struct winesync_wait_args { struct winesync_sem_args) #define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 1, \ struct winesync_mutex_args) +#define WINESYNC_IOC_GET_SEM _IOW (WINESYNC_IOC_BASE, 2, __u32) #define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ struct winesync_sem_args) #define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ From 1535e93822c1d30e595f542a9ef0be97c9a255a7 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 17:21:26 -0600 Subject: [PATCH 44/48] winesync: Introduce WINESYNC_IOC_PULSE_SEM. Signed-off-by: Kai Krakow --- drivers/misc/winesync.c | 13 +++++++++++-- include/uapi/linux/winesync.h | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c index d4779d09ab91d1..a4c1dbc2a33eba 100644 --- a/drivers/misc/winesync.c +++ b/drivers/misc/winesync.c @@ -454,7 +454,8 @@ static int put_sem_state(struct winesync_obj *sem, __u32 count) return 0; } -static int winesync_put_sem(struct winesync_device *dev, void __user *argp) +static int winesync_put_sem(struct winesync_device *dev, void __user *argp, + bool pulse) { struct winesync_sem_args __user *user_args = argp; struct winesync_sem_args args; @@ -484,6 +485,9 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) try_wake_any_sem(sem); } + if (pulse) + sem->u.sem.count = 0; + spin_unlock(&sem->lock); spin_unlock(&dev->wait_all_lock); } else { @@ -494,6 +498,9 @@ static int winesync_put_sem(struct winesync_device *dev, void __user *argp) if (!ret) try_wake_any_sem(sem); + if (pulse) + sem->u.sem.count = 0; + spin_unlock(&sem->lock); } @@ -985,7 +992,9 @@ static long winesync_char_ioctl(struct file *file, unsigned int cmd, case WINESYNC_IOC_GET_SEM: return winesync_get_sem(dev, argp); case WINESYNC_IOC_PUT_SEM: - return winesync_put_sem(dev, argp); + return winesync_put_sem(dev, argp, false); + case WINESYNC_IOC_PULSE_SEM: + return winesync_put_sem(dev, argp, true); case WINESYNC_IOC_PUT_MUTEX: return winesync_put_mutex(dev, argp); case WINESYNC_IOC_READ_SEM: diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h index 96bceacb809126..c16f423dad48e0 100644 --- a/include/uapi/linux/winesync.h +++ b/include/uapi/linux/winesync.h @@ -41,6 +41,8 @@ struct winesync_wait_args { #define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 1, \ struct winesync_mutex_args) #define WINESYNC_IOC_GET_SEM _IOW (WINESYNC_IOC_BASE, 2, __u32) +#define WINESYNC_IOC_PULSE_SEM _IOWR(WINESYNC_IOC_BASE, 3, \ + struct winesync_sem_args) #define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 5, \ struct winesync_sem_args) #define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ From 04cb57652a02f5a9c2c77ef74f35aa38628e7489 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 17:21:47 -0600 Subject: [PATCH 45/48] doc: Document WINESYNC_IOC_GET_SEM and WINESYNC_IOC_PULSE_SEM. Signed-off-by: Kai Krakow --- Documentation/userspace-api/winesync.rst | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst index 3c5e986ddd8c1f..c980eab9f2ee3c 100644 --- a/Documentation/userspace-api/winesync.rst +++ b/Documentation/userspace-api/winesync.rst @@ -157,6 +157,46 @@ The ioctls are as follows: semaphore will be woken and the semaphore's count decremented appropriately. +.. c:macro:: WINESYNC_IOC_PULSE_SEM + + This operation is identical to ``WINESYNC_IOC_PUT_SEM``, with one + notable exception: the semaphore is always left in an *unsignaled* + state, regardless of the initial count or the count added by the + ioctl. That is, the count after a pulse operation will always be + zero. The entire operation is atomic. + + Hence, if the semaphore was created with the + ``WINESYNC_SEM_GETONWAIT`` flag set, and an unsignaled semaphore is + "pulsed" with a count of 2, at most two eligible threads (i.e. + threads not otherwise constrained due to ``WINESYNC_IOC_WAIT_ALL``) + will be woken up, and any others will remain sleeping. If less than + two eligible threads are waiting on the semaphore, all of them will + be woken up, and the semaphore's count will remain at zero. On the + other hand, if the semaphore was created without the + ``WINESYNC_SEM_GETONWAIT``, all eligible threads will be woken up, + making ``count`` effectively redundant. In either case, a + simultaneous ``WINESYNC_IOC_READ_SEM`` ioctl from another thread + will always report a count of zero. + + If adding ``count`` to the semaphore's current count would raise the + latter past the semaphore's maximum count, the ioctl fails with + ``EOVERFLOW``. However, in this case the semaphore's count will + still be reset to zero. + +.. c:macro:: WINESYNC_IOC_GET_SEM + + Attempt to acquire a semaphore object. Takes an input-only pointer + to a 32-bit integer denoting the semaphore to acquire. + + This operation does not block. If the semaphore's count was zero, it + fails with ``EWOULDBLOCK``. Otherwise, the semaphore's count is + decremented by one. The behaviour of this operation is unaffected by + whether the semaphore was created with the + ``WINESYNC_SEM_GETONWAIT`` flag set. + + The operation is atomic and totally ordered with respect to other + operations on the same semaphore. + .. c:macro:: WINESYNC_IOC_PUT_MUTEX Release a mutex object. Takes a pointer to struct From e871a00f96762e99403475480702875787292b4b Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 17:23:16 -0600 Subject: [PATCH 46/48] selftests: winesync: Add tests for semaphores without WINESYNC_SEM_GETONWAIT and for WINESYNC_IOC_GET_SEM. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 97719feb625e5a..01d638f99f18b6 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -26,6 +26,153 @@ TEST(semaphore_state) fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ASSERT_LE(0, fd); + sem_args.count = 3; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + sem_args.flags = 0; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + sem_args.count = 2; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + sem_args.flags = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + EXPECT_EQ(0, sem_args.flags); + + sem_args.count = 0; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)&sem_args.sem; + wait_args.count = 1; + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EWOULDBLOCK, errno); + + sem_args.count = 3; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(semaphore_state_getonwait) +{ + struct winesync_wait_args wait_args; + struct winesync_sem_args sem_args; + struct timespec timeout; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + sem_args.count = 3; sem_args.max = 2; sem_args.sem = 0xdeadbeef; @@ -145,6 +292,44 @@ TEST(semaphore_state) EXPECT_EQ(1, sem_args.count); EXPECT_EQ(2, sem_args.max); + /* Test GET. */ + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EWOULDBLOCK, errno); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(0, ret); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(2, sem_args.max); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); EXPECT_EQ(0, ret); @@ -770,6 +955,10 @@ TEST(invalid_objects) EXPECT_EQ(-1, ret); EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_GET_SEM, &sem_args.sem); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); EXPECT_EQ(-1, ret); EXPECT_EQ(EINVAL, errno); From d344c338633f47738a90ad6ea3eed886ffbda519 Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Fri, 5 Mar 2021 17:23:35 -0600 Subject: [PATCH 47/48] selftests: winesync: Add tests for WINESYNC_IOC_PULSE_SEM. Signed-off-by: Kai Krakow --- .../selftests/drivers/winesync/winesync.c | 161 +++++++++++++++++- 1 file changed, 159 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c index 01d638f99f18b6..c4ac22b7fdb7dd 100644 --- a/tools/testing/selftests/drivers/winesync/winesync.c +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -155,6 +155,61 @@ TEST(semaphore_state) EXPECT_EQ(1, sem_args.count); EXPECT_EQ(2, sem_args.max); + /* Test PULSE. */ + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); EXPECT_EQ(0, ret); @@ -330,6 +385,61 @@ TEST(semaphore_state_getonwait) EXPECT_EQ(1, sem_args.count); EXPECT_EQ(2, sem_args.max); + /* Test PULSE. */ + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + sem_args.count = 0xdeadbeef; + sem_args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + EXPECT_EQ(2, sem_args.max); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); EXPECT_EQ(0, ret); @@ -951,6 +1061,10 @@ TEST(invalid_objects) EXPECT_EQ(-1, ret); EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); EXPECT_EQ(-1, ret); EXPECT_EQ(EINVAL, errno); @@ -1136,6 +1250,30 @@ TEST(wake_any) EXPECT_EQ(0, thread_args.ret); EXPECT_EQ(0, wait_args.index); + /* test waking the semaphore via pulse */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.owner = 456; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(0, wait_args.index); + /* test waking the mutex */ /* first grab it again for owner 123 */ @@ -1281,14 +1419,14 @@ TEST(wake_all) EXPECT_EQ(0, mutex_args.count); EXPECT_EQ(0, mutex_args.owner); - sem_args.count = 2; + sem_args.count = 1; ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); EXPECT_EQ(0, ret); EXPECT_EQ(0, sem_args.count); ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); EXPECT_EQ(0, ret); - EXPECT_EQ(1, sem_args.count); + EXPECT_EQ(0, sem_args.count); ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); EXPECT_EQ(0, ret); @@ -1299,6 +1437,25 @@ TEST(wake_all) EXPECT_EQ(0, ret); EXPECT_EQ(0, thread_args.ret); + /* test waking the semaphore via pulse */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.owner = 456; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PULSE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + /* delete an object while it's being waited on */ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); From 53ee04a65b6f7599742cdf17ab59919ac745a39e Mon Sep 17 00:00:00 2001 From: Paul Gofman Date: Wed, 6 May 2020 14:37:44 +0300 Subject: [PATCH 48/48] mm: Support soft dirty flag reset for VA range. --- fs/proc/task_mmu.c | 129 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 26 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 3cec6fbef725ed..7c7865028f1066 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1032,6 +1032,8 @@ enum clear_refs_types { struct clear_refs_private { enum clear_refs_types type; + unsigned long start, end; + bool clear_range; }; #ifdef CONFIG_MEM_SOFT_DIRTY @@ -1125,6 +1127,8 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, spinlock_t *ptl; struct page *page; + BUG_ON(addr < cp->start || end > cp->end); + ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1181,9 +1185,11 @@ static int clear_refs_test_walk(unsigned long start, unsigned long end, struct clear_refs_private *cp = walk->private; struct vm_area_struct *vma = walk->vma; - if (vma->vm_flags & VM_PFNMAP) + if (!cp->clear_range && (vma->vm_flags & VM_PFNMAP)) return 1; + BUG_ON(start < cp->start || end > cp->end); + /* * Writing 1 to /proc/pid/clear_refs affects all pages. * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. @@ -1206,10 +1212,12 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct task_struct *task; - char buffer[PROC_NUMBUF]; + char buffer[18]; struct mm_struct *mm; struct vm_area_struct *vma; enum clear_refs_types type; + unsigned long start, end; + bool clear_range; int itype; int rv; @@ -1218,12 +1226,34 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, count = sizeof(buffer) - 1; if (copy_from_user(buffer, buf, count)) return -EFAULT; - rv = kstrtoint(strstrip(buffer), 10, &itype); - if (rv < 0) - return rv; - type = (enum clear_refs_types)itype; - if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) - return -EINVAL; + + if (buffer[0] == '6') + { + static int once; + + if (!once++) + printk(KERN_DEBUG "task_mmu: Using POC clear refs range implementation.\n"); + + if (count != 17) + return -EINVAL; + + type = CLEAR_REFS_SOFT_DIRTY; + start = *(unsigned long *)(buffer + 1); + end = *(unsigned long *)(buffer + 1 + 8); + } + else + { + rv = kstrtoint(strstrip(buffer), 10, &itype); + if (rv < 0) + return rv; + type = (enum clear_refs_types)itype; + + if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) + return -EINVAL; + + start = 0; + end = -1UL; + } task = get_proc_task(file_inode(file)); if (!task) @@ -1235,41 +1265,87 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, .type = type, }; - if (mmap_write_lock_killable(mm)) { - count = -EINTR; - goto out_mm; + if (start || end != -1UL) + { + start = min(start, mm->highest_vm_end) & PAGE_MASK; + end = min(end, mm->highest_vm_end) & PAGE_MASK; + + if (start >= end) + { + count = -EINVAL; + goto out_mm; + } + clear_range = true; } + else + { + clear_range = false; + } + + cp.start = start; + cp.end = end; + cp.clear_range = clear_range; + if (type == CLEAR_REFS_MM_HIWATER_RSS) { + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + /* * Writing 5 to /proc/pid/clear_refs resets the peak * resident set size to this mm's current rss value. */ reset_mm_hiwater_rss(mm); - goto out_unlock; + mmap_write_unlock(mm); + goto out_mm; } if (type == CLEAR_REFS_SOFT_DIRTY) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (!(vma->vm_flags & VM_SOFTDIRTY)) - continue; - vma->vm_flags &= ~VM_SOFTDIRTY; - vma_set_page_prot(vma); + if (mmap_read_lock_killable(mm)) { + count = -EINTR; + goto out_mm; } - + if (!clear_range) + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (!(vma->vm_flags & VM_SOFTDIRTY)) + continue; + mmap_read_unlock(mm); + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + for (vma = mm->mmap; vma; vma = vma->vm_next) { + vma->vm_flags &= ~VM_SOFTDIRTY; + vma_set_page_prot(vma); + } + mmap_write_downgrade(mm); + break; + } inc_tlb_flush_pending(mm); mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, - 0, NULL, mm, 0, -1UL); + 0, NULL, mm, start, end); mmu_notifier_invalidate_range_start(&range); } - walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, + else + { + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + } + walk_page_range(mm, start, end == -1UL ? mm->highest_vm_end : end, &clear_refs_walk_ops, &cp); if (type == CLEAR_REFS_SOFT_DIRTY) { mmu_notifier_invalidate_range_end(&range); flush_tlb_mm(mm); dec_tlb_flush_pending(mm); + mmap_read_unlock(mm); + } + else + { + mmap_write_unlock(mm); } -out_unlock: - mmap_write_unlock(mm); out_mm: mmput(mm); } @@ -1301,6 +1377,7 @@ struct pagemapread { #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) #define PM_SOFT_DIRTY BIT_ULL(55) #define PM_MMAP_EXCLUSIVE BIT_ULL(56) +#define PM_SOFT_DIRTY_PAGE BIT_ULL(57) #define PM_FILE BIT_ULL(61) #define PM_SWAP BIT_ULL(62) #define PM_PRESENT BIT_ULL(63) @@ -1373,11 +1450,11 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; } else if (is_swap_pte(pte)) { swp_entry_t entry; if (pte_swp_soft_dirty(pte)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; entry = pte_to_swp_entry(pte); if (pm->show_pfn) frame = swp_type(entry) | @@ -1424,7 +1501,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; if (pm->show_pfn) frame = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); @@ -1442,7 +1519,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, } flags |= PM_SWAP; if (pmd_swp_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; VM_BUG_ON(!is_pmd_migration_entry(pmd)); page = migration_entry_to_page(entry); }