caprevoke: stop and scan all procs sharing our vmspace

Maintain a list of processes sharing each vmspace so they can be scanned
efficently.

In addition to thread_single() on the revoking process, add
stop_vmspace_proc() which stops all other processes that share our
vmspace following the model of stop_all_proc().  For each process use a
new thread_single(SINGLE_VMSPACE) which acts like SINGLE_BOUNDRY except
that like SINGLE_ALLPROC it suspends all threads in the process and does
not expect proc to be curproc.

sys/kern/init_main.c

@@ -627,7 +627,10 @@ proc0_init(void *dummy __unused)
 
  /* Allocate a prototype map so we have something to fork. */
  p->p_vmspace = &vmspace0;
+ mtx_init(&vmspace0.vm_mtx, "vmspace", NULL, MTX_DEF);
  refcount_init(&vmspace0.vm_refcnt, 1);
+ LIST_INIT(&vmspace0.vm_proclist);
+ LIST_INSERT_HEAD(&vmspace0.vm_proclist, p, p_vm_proclist);
  pmap_pinit0(vmspace_pmap(&vmspace0));
 
  /*

sys/kern/kern_cheri_revoke.c

@@ -125,6 +125,7 @@ kern_cheri_revoke(struct thread *td, int flags,
 #else
  struct cheri_revoke_stats *crstp = NULL;
 #endif
+ struct proc *proc;
  struct vmspace *vm;
  vm_map_t vmm;
  struct vm_cheri_revoke_cookie vmcrc;
@@ -426,6 +427,7 @@ kern_cheri_revoke(struct thread *td, int flags,
  return ERESTART;
  }
  }
+ stop_vmspace_proc(td->td_proc);
 
  /*
  * Drop the process lock *then* iterate the threads in this
@@ -448,20 +450,27 @@ kern_cheri_revoke(struct thread *td, int flags,
  * This also risks the use of ptrace() to expose to userspace
  * the trap frame of a stalled thread that has not yet scanned
  * itself. Yick.
+ *
+ * XXX-BD: is it safe to walk the proc list and run the
+ * revokers or do we need to PHOLD them first?
  */
 
  _PHOLD(td->td_proc);
  PROC_UNLOCK(td->td_proc);
 
  /* Per-thread kernel hoarders */
- FOREACH_THREAD_IN_PROC (td->td_proc, ptd) {
- cheri_revoke_td_frame(ptd, &vmcrc);
- sigaltstack_cheri_revoke(ptd, &vmcrc);
+ LIST_FOREACH(proc, &td->td_proc->p_vmspace->vm_proclist,
+ p_vm_proclist) {
+ FOREACH_THREAD_IN_PROC (proc, ptd) {
+ cheri_revoke_td_frame(ptd, &vmcrc);
+ sigaltstack_cheri_revoke(ptd, &vmcrc);
+ }
  }
  }
 
  /* Per-process kernel hoarders */
- cheri_revoke_hoarders(td->td_proc, &vmcrc);
+ LIST_FOREACH(proc, &td->td_proc->p_vmspace->vm_proclist, p_vm_proclist)
+ cheri_revoke_hoarders(proc, &vmcrc);
 
  switch(myst) {
  default:
@@ -526,6 +535,7 @@ kern_cheri_revoke(struct thread *td, int flags,
 
  PROC_LOCK(td->td_proc);
  _PRELE(td->td_proc);
+ resume_vmspace_proc(td->td_proc);
  if ((td->td_proc->p_flag & P_HADTHREADS) != 0) {
  thread_single_end(td->td_proc, SINGLE_BOUNDARY);
  }

sys/kern/kern_proc.c

@@ -3854,6 +3854,127 @@ resume_all_proc(void)
  stop_all_proc_unblock();
 }
 
+/*
+ * stop_vmspace_proc() stops all proceses which share a vmspace with
+ * curproc. It should be run after curproc has entered thread_single.
+ */
+void
+stop_vmspace_proc(struct proc* cp)
+{
+ struct proc *p;
+ struct vmspace *vm;
+ int r, gen;
+ bool restart, seen_stopped, seen_exiting, stopped_some;
+
+ /*
+ * XXX-BD: Prevent multiple callers on a vmspace.
+ * Maybe atomic CAS on a field in vmspace?
+ */
+
+ vm = cp->p_vmspace;
+vmspace_loop:
+ VMSPACE_LOCK(vm);
+ /*
+ * XXX: allproc_gen should work, but should vmspaces have a generation?
+ */
+ gen = allproc_gen;
+ seen_exiting = seen_stopped = stopped_some = restart = false;
+ LIST_REMOVE(cp, p_vm_proclist);
+ LIST_INSERT_HEAD(&vm->vm_proclist, cp, p_vm_proclist);
+ for (;;) {
+ p = LIST_NEXT(cp, p_vm_proclist);
+ if (p == NULL)
+ break;
+ LIST_REMOVE(cp, p_vm_proclist);
+ LIST_INSERT_AFTER(p, cp, p_vm_proclist);
+ PROC_LOCK(p);
+ if ((p->p_flag & (P_KPROC | P_SYSTEM | P_TOTAL_STOP)) != 0) {
+ PROC_UNLOCK(p);
+ continue;
+ }
+ if ((p->p_flag2 & P2_WEXIT) != 0) {
+ seen_exiting = true;
+ PROC_UNLOCK(p);
+ continue;
+ }
+ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
+ /*
+ * Stopped processes are tolerated when there
+ * are no other processes which might continue
+ * them. P_STOPPED_SINGLE but not
+ * P_TOTAL_STOP process still has at least one
+ * thread running.
+ */
+ seen_stopped = true;
+ PROC_UNLOCK(p);
+ continue;
+ }
+ VMSPACE_UNLOCK(vm);
+ _PHOLD(p);
+ r = thread_single(p, SINGLE_VMSPACE);
+ if (r != 0)
+ restart = true;
+ else
+ stopped_some = true;
+ _PRELE(p);
+ PROC_UNLOCK(p);
+ VMSPACE_LOCK(vm);
+ }
+ /* Catch forked children we did not see in iteration. */
+ if (gen != allproc_gen)
+ restart = true;
+ VMSPACE_UNLOCK(vm);
+ if (restart || stopped_some || seen_exiting || seen_stopped) {
+ kern_yield(PRI_USER);
+ goto vmspace_loop;
+ }
+}
+
+void
+resume_vmspace_proc(struct proc *cp)
+{
+ struct proc *p;
+ struct vmspace *vm;
+
+ vm = cp->p_vmspace;
+
+ VMSPACE_LOCK(vm);
+again:
+ LIST_REMOVE(cp, p_vm_proclist);
+ LIST_INSERT_HEAD(&vm->vm_proclist, cp, p_vm_proclist);
+ for (;;) {
+ p = LIST_NEXT(cp, p_vm_proclist);
+ if (p == NULL)
+ break;
+ LIST_REMOVE(cp, p_vm_proclist);
+ LIST_INSERT_AFTER(p, cp, p_vm_proclist);
+ PROC_LOCK(p);
+ if (p->p_vmspace != cp->p_vmspace) {
+ PROC_UNLOCK(p);
+ continue;
+ }
+ if ((p->p_flag & P_TOTAL_STOP) != 0) {
+ VMSPACE_UNLOCK(vm);
+ _PHOLD(p);
+ thread_single_end(p, SINGLE_VMSPACE);
+ _PRELE(p);
+ PROC_UNLOCK(p);
+ VMSPACE_LOCK(vm);
+ } else {
+ PROC_UNLOCK(p);
+ }
+ }
+ /* Did the loop above missed any stopped process ? */
+ FOREACH_PROC_IN_SYSTEM(p) {
+ /* No need for proc lock. */
+ if ((p->p_flag & P_TOTAL_STOP) != 0)
+ goto again;
+ }
+ VMSPACE_UNLOCK(vm);
+
+ /* See "XXX-BD: Prevent multiple callers on a vmspace" above */
+}
+
 /* #define TOTAL_STOP_DEBUG 1 */
 #ifdef TOTAL_STOP_DEBUG
 volatile static int ap_resume;

sys/kern/kern_thread.c

@@ -1097,7 +1097,7 @@ calc_remaining(struct proc *p, int mode)
  PROC_SLOCK_ASSERT(p, MA_OWNED);
  if (mode == SINGLE_EXIT)
  remaining = p->p_numthreads;
- else if (mode == SINGLE_BOUNDARY)
+ else if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE)
  remaining = p->p_numthreads - p->p_boundary_count;
  else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC)
  remaining = p->p_numthreads - p->p_suspcount;
@@ -1143,6 +1143,7 @@ weed_inhib(int mode, struct thread *td2, struct proc *p)
  break;
  case SINGLE_BOUNDARY:
  case SINGLE_NO_EXIT:
+ case SINGLE_VMSPACE:
  if (TD_IS_SUSPENDED(td2) &&
  (td2->td_flags & TDF_BOUNDARY) == 0) {
  wakeup_swapper |= thread_unsuspend_one(td2, p, false);
@@ -1163,8 +1164,8 @@ weed_inhib(int mode, struct thread *td2, struct proc *p)
  * boundary, TDF_ALLPROCSUSP is used to avoid immediate
  * un-suspend.
  */
- if (TD_IS_SUSPENDED(td2) && (td2->td_flags &
- TDF_ALLPROCSUSP) == 0) {
+ if (TD_IS_SUSPENDED(td2) &&
+ (td2->td_flags & TDF_ALLPROCSUSP) == 0) {
  wakeup_swapper |= thread_unsuspend_one(td2, p, false);
  thread_lock(td2);
  goto restart;
@@ -1204,7 +1205,8 @@ thread_single(struct proc *p, int mode)
 
  td = curthread;
  KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
- mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
+ mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT ||
+ mode == SINGLE_VMSPACE,
  ("invalid mode %d", mode));
  /*
  * If allowing non-ALLPROC singlethreading for non-curproc
@@ -1213,7 +1215,8 @@ thread_single(struct proc *p, int mode)
  * this is not implemented because it is not used.
  */
  KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) ||
- (mode != SINGLE_ALLPROC && td->td_proc == p),
+ ((mode != SINGLE_ALLPROC || mode != SINGLE_VMSPACE) &&
+ td->td_proc == p),
  ("mode %d proc %p curproc %p", mode, p, td->td_proc));
  mtx_assert(&Giant, MA_NOTOWNED);
  PROC_LOCK_ASSERT(p, MA_OWNED);
@@ -1238,7 +1241,7 @@ thread_single(struct proc *p, int mode)
  p->p_flag &= ~P_SINGLE_BOUNDARY;
  } else {
  p->p_flag &= ~P_SINGLE_EXIT;
- if (mode == SINGLE_BOUNDARY)
+ if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE)
  p->p_flag |= P_SINGLE_BOUNDARY;
  else
  p->p_flag &= ~P_SINGLE_BOUNDARY;
@@ -1306,7 +1309,7 @@ thread_single(struct proc *p, int mode)
  PROC_LOCK(p);
  PROC_SLOCK(p);
  }
- } else if (mode == SINGLE_BOUNDARY) {
+ } else if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE) {
  /*
  * Wait until all suspended threads are removed from
  * the processors. The thread_suspend_check()

sys/sys/proc.h

@@ -682,6 +682,8 @@ struct proc {
  (if I am reaper). */
  LIST_ENTRY(proc) p_reapsibling; /* (e) List of siblings - descendants of
  the same reaper. */
+ LIST_ENTRY(proc) p_vm_proclist; /* (b) List of processes sharing
+ p_vmspace */
  struct mtx p_mtx; /* (n) Lock for this struct. */
  struct mtx p_statmtx; /* Lock for the stats */
  struct mtx p_itimmtx; /* Lock for the virt/prof timers */
@@ -935,6 +937,7 @@ struct proc {
 #define SINGLE_EXIT 1
 #define SINGLE_BOUNDARY 2
 #define SINGLE_ALLPROC 3
+#define SINGLE_VMSPACE 4
 
 #ifdef MALLOC_DECLARE
 MALLOC_DECLARE(M_PARGS);
@@ -1297,6 +1300,8 @@ void stop_all_proc_unblock(void);
 
 void stop_all_proc(void);
 void resume_all_proc(void);
+void stop_vmspace_proc(struct proc *cp);
+void resume_vmspace_proc(struct proc *cp);
 
 static __inline int
 curthread_pflags_set(int flags)

sys/vm/vm_glue.c

@@ -585,6 +585,7 @@ vm_forkproc(struct thread *td, struct proc *p2, struct thread *td2,
  if (p1->p_vmspace->vm_shm)
  shmfork(p1, p2);
  }
+ vmspace_insert_proc(p2->p_vmspace, p2);
 
  /*
  * cpu_fork will copy and update the pcb, set up the kernel stack,