OS-4151 setbrand hooks should be sane during fork

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Joshua M. Clulow <jmc@joyent.com>

usr/src/uts/common/brand/lx/io/lx_netlink.c

@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright (c) 2014 Joyent, Inc.  All rights reserved.
+ * Copyright 2015 Joyent, Inc.
  */
 
 /*
@@ -392,7 +392,7 @@ lx_netlink_getsockname(sock_lower_handle_t handle, struct sockaddr *sa,
 	/*
 	 * Make sure our lies are consistent with the lies told by other liars.
 	 */
-	if (p->p_brand != &native_brand && curthread != p->p_agenttp) {
+	if (PROC_IS_BRANDED(p) && curthread != p->p_agenttp) {
 		lxsa->lxnl_family = LX_AF_NETLINK;
 	} else {
 		lxsa->lxnl_family = AF_LX_NETLINK;

usr/src/uts/common/brand/lx/os/lx_brand.c

@@ -243,6 +243,7 @@ struct brand_ops lx_brops = {
 	lx_proc_exit,			/* b_proc_exit */
 	lx_exec,			/* b_exec */
 	lx_setrval,			/* b_lwp_setrval */
+	lx_brandlwp,			/* b_brandlwp */
 	lx_initlwp,			/* b_initlwp */
 	lx_forklwp,			/* b_forklwp */
 	lx_freelwp,			/* b_freelwp */
@@ -312,22 +313,8 @@ lx_proc_exit(proc_t *p)
 void
 lx_setbrand(proc_t *p)
 {
-	kthread_t *t = p->p_tlist;
-	int err;
-
-	ASSERT(p->p_brand_data == NULL);
-	ASSERT(ttolxlwp(curthread) == NULL);
-
-	p->p_brand_data = kmem_zalloc(sizeof (struct lx_proc_data), KM_SLEEP);
+	/* Send SIGCHLD to parent by default when child exits */
 	ptolxproc(p)->l_signal = stol_signo[SIGCHLD];
-
-	/*
-	 * This routine can only be called for single-threaded processes.
-	 * Since lx_initlwp() can only fail if we run out of PIDs for
-	 * multithreaded processes, we know that this can never fail.
-	 */
-	err = lx_initlwp(t->t_lwp);
-	ASSERT(err == 0);
 }
 
 /* ARGSUSED */
@@ -1308,18 +1295,16 @@ lx_setid_clear(vattr_t *vap, cred_t *cr)
 void
 lx_copy_procdata(proc_t *child, proc_t *parent)
 {
-	lx_proc_data_t *cpd, *ppd;
+	lx_proc_data_t *cpd = child->p_brand_data;
+	lx_proc_data_t *ppd = parent->p_brand_data;
 
-	ppd = parent->p_brand_data;
+	VERIFY(parent->p_brand == &lx_brand);
+	VERIFY(child->p_brand == &lx_brand);
+	VERIFY(ppd != NULL);
+	VERIFY(cpd != NULL);
 
-	ASSERT(ppd != NULL);
-	ASSERT(parent->p_brand == &lx_brand);
-
-	cpd = kmem_alloc(sizeof (lx_proc_data_t), KM_SLEEP);
 	*cpd = *ppd;
 
-	child->p_brand_data = cpd;
-
 	cpd->l_fake_limits[LX_RLFAKE_LOCKS].rlim_cur = LX_RLIM64_INFINITY;
 	cpd->l_fake_limits[LX_RLFAKE_LOCKS].rlim_max = LX_RLIM64_INFINITY;
 

usr/src/uts/common/brand/lx/os/lx_misc.c

@@ -51,15 +51,8 @@
 #include <sys/sunddi.h>
 
 /* Linux specific functions and definitions */
-void lx_setrval(klwp_t *, int, int);
-void lx_exec();
-int lx_initlwp(klwp_t *);
-void lx_forklwp(klwp_t *, klwp_t *);
-void lx_exitlwp(klwp_t *);
-void lx_freelwp(klwp_t *);
 static void lx_save(klwp_t *);
 static void lx_restore(klwp_t *);
-extern void lx_ptrace_free(proc_t *);
 
 /*
  * Set the return code for the forked child, always zero
@@ -82,7 +75,6 @@ lx_exec()
 	proc_t *p = ttoproc(curthread);
 	lx_proc_data_t *pd = ptolxproc(p);
 	struct regs *rp = lwptoregs(lwp);
-	int err;
 
 	/*
 	 * Any l_handler handlers set as a result of B_REGISTER are now
@@ -91,25 +83,10 @@ lx_exec()
 	pd->l_handler = NULL;
 
 	/*
-	 * There are two mutually exclusive special cases we need to
-	 * address.  First, if this was a native process prior to this
-	 * exec(), then this lwp won't have its brand-specific data
-	 * initialized and it won't be assigned a Linux PID yet.  Second,
-	 * if this was a multi-threaded Linux process and this lwp wasn't
-	 * the main lwp, then we need to make its Solaris and Linux PIDS
-	 * match.
+	 * If this was a multi-threaded Linux process and this lwp wasn't the
+	 * main lwp, then we need to make its Illumos and Linux PIDs match.
 	 */
-	if (lwpd == NULL) {
-		err = lx_initlwp(lwp);
-		/*
-		 * Only possible failure from this routine should be an
-		 * inability to allocate a new PID.  Since single-threaded
-		 * processes don't need a new PID, we should never hit this
-		 * error.
-		 */
-		ASSERT(err == 0);
-		lwpd = lwptolxlwp(lwp);
-	} else if (curthread->t_tid != 1) {
+	if (curthread->t_tid != 1) {
 		lx_pid_reassign(curthread);
 	}
 
@@ -260,8 +237,7 @@ lx_freelwp(klwp_t *lwp)
 	(void) removectx(lwptot(lwp), lwp, lx_save, lx_restore, NULL, NULL,
 	    lx_save, NULL);
 	if (lwpd->br_pid != 0) {
-		lx_pid_rele(lwptoproc(lwp)->p_pid,
-		    lwptot(lwp)->t_tid);
+		lx_pid_rele(lwptoproc(lwp)->p_pid, lwptot(lwp)->t_tid);
 	}
 
 	/*
@@ -276,20 +252,21 @@ lx_freelwp(klwp_t *lwp)
 }
 
 int
-lx_initlwp(klwp_t *lwp)
+lx_brandlwp(klwp_t *lwp)
 {
 	lx_lwp_data_t *lwpd;
 	lx_lwp_data_t *plwpd = ttolxlwp(curthread);
 	kthread_t *tp = lwptot(lwp);
 
+	VERIFY(lwp->lwp_brand == NULL);
+
 	lwpd = kmem_zalloc(sizeof (struct lx_lwp_data), KM_SLEEP);
 	lwpd->br_exitwhy = CLD_EXITED;
 	lwpd->br_lwp = lwp;
 	lwpd->br_clear_ctidp = NULL;
 	lwpd->br_set_ctidp = NULL;
 	lwpd->br_signal = 0;
 	lwpd->br_stack_mode = LX_STACK_MODE_PREINIT;
-
 	/*
 	 * lwpd->br_affinitymask was zeroed by kmem_zalloc()
 	 * as was lwpd->br_scall_args and lwpd->br_args_size.
@@ -330,20 +307,25 @@ lx_initlwp(klwp_t *lwp)
 	installctx(lwptot(lwp), lwp, lx_save, lx_restore, NULL, NULL,
 	    lx_save, NULL);
 
+	/*
+	 * Install branded system call hook for this LWP:
+	 */
+	lwp->lwp_brand_syscall = lx_syscall_enter;
+	return (0);
+}
+
+void
+lx_initlwp(klwp_t *lwp)
+{
+	lx_lwp_data_t *lwpd = lwptolxlwp(lwp);
+	lx_lwp_data_t *plwpd = ttolxlwp(curthread);
 	/*
 	 * If the parent LWP has a ptrace(2) tracer, the new LWP may
 	 * need to inherit that same tracer.
 	 */
 	if (plwpd != NULL) {
 		lx_ptrace_inherit_tracer(plwpd, lwpd);
 	}
-
-	/*
-	 * Install branded system call hook for this LWP:
-	 */
-	lwp->lwp_brand_syscall = lx_syscall_enter;
-
-	return (0);
 }
 
 /*

usr/src/uts/common/brand/lx/sys/lx_misc.h

@@ -28,7 +28,8 @@ extern void lx_setrval(klwp_t *, int, int);
 extern void lx_exec();
 extern void lx_exitlwp(klwp_t *);
 extern void lx_freelwp(klwp_t *);
-extern int lx_initlwp(klwp_t *);
+extern int lx_brandlwp(klwp_t *);
+extern void lx_initlwp(klwp_t *);
 extern void lx_forklwp(klwp_t *, klwp_t *);
 
 extern void lx_set_gdt(int, user_desc_t *);

usr/src/uts/common/brand/sn1/sn1_brand.c

@@ -53,7 +53,7 @@ int	sn1_brandsys(int, int64_t *, uintptr_t, uintptr_t, uintptr_t,
 void	sn1_copy_procdata(proc_t *, proc_t *);
 void	sn1_proc_exit(struct proc *);
 void	sn1_exec();
-int	sn1_initlwp(klwp_t *);
+int	sn1_brandlwp(klwp_t *);
 void	sn1_forklwp(klwp_t *, klwp_t *);
 void	sn1_freelwp(klwp_t *);
 void	sn1_lwpexit(klwp_t *);
@@ -72,7 +72,8 @@ struct brand_ops sn1_brops = {
 	sn1_proc_exit,			/* b_proc_exit */
 	sn1_exec,			/* b_exec */
 	lwp_setrval,			/* b_lwp_setrval */
-	sn1_initlwp,			/* b_initlwp */
+	sn1_brandlwp,			/* b_brandlwp */
+	NULL,				/* b_initlwp */
 	sn1_forklwp,			/* b_forklwp */
 	sn1_freelwp,			/* b_freelwp */
 	sn1_lwpexit,			/* b_lwpexit */
@@ -210,9 +211,9 @@ sn1_exec()
 }
 
 int
-sn1_initlwp(klwp_t *l)
+sn1_brandlwp(klwp_t *l)
 {
-	return (brand_solaris_initlwp(l, &sn1_brand));
+	return (brand_solaris_brandlwp(l, &sn1_brand));
 }
 
 void

usr/src/uts/common/brand/solaris10/s10_brand.c

@@ -56,7 +56,7 @@ int	s10_brandsys(int, int64_t *, uintptr_t, uintptr_t, uintptr_t,
 void	s10_copy_procdata(proc_t *, proc_t *);
 void	s10_proc_exit(struct proc *);
 void	s10_exec();
-int	s10_initlwp(klwp_t *);
+int	s10_brandlwp(klwp_t *);
 void	s10_forklwp(klwp_t *, klwp_t *);
 void	s10_freelwp(klwp_t *);
 void	s10_lwpexit(klwp_t *);
@@ -77,7 +77,8 @@ struct brand_ops s10_brops = {
 	s10_proc_exit,			/* b_proc_exit */
 	s10_exec,			/* b_exec */
 	lwp_setrval,			/* b_lwp_setrval */
-	s10_initlwp,			/* b_initlwp */
+	s10_brandlwp,			/* b_brandlwp */
+	NULL,				/* b_initlwp */
 	s10_forklwp,			/* b_forklwp */
 	s10_freelwp,			/* b_freelwp */
 	s10_lwpexit,			/* b_lwpexit */
@@ -365,9 +366,9 @@ s10_exec()
 }
 
 int
-s10_initlwp(klwp_t *l)
+s10_brandlwp(klwp_t *l)
 {
-	return (brand_solaris_initlwp(l, &s10_brand));
+	return (brand_solaris_brandlwp(l, &s10_brand));
 }
 
 void

usr/src/uts/common/os/brand.c

@@ -55,7 +55,7 @@ brand_t native_brand = {
 		"native",
 		NULL,
 		&native_mach_ops,
-		0,
+		0
 };
 
 /*
@@ -316,35 +316,54 @@ void
 brand_setbrand(proc_t *p)
 {
 	brand_t *bp = p->p_zone->zone_brand;
+	void *brand_data = NULL;
 
-	ASSERT(bp != NULL);
-	ASSERT(p->p_brand == &native_brand);
+	VERIFY(MUTEX_NOT_HELD(&p->p_lock));
+	VERIFY(bp != NULL);
 
 	/*
-	 * We should only be called from exec(), when we know the process
-	 * is single-threaded.
+	 * We should only be called from exec() or getproc(), when we know the
+	 * process has 0 or 1 threads.
 	 */
-	ASSERT(p->p_tlist == p->p_tlist->t_forw);
+	VERIFY((p->p_tlist == NULL) || (p->p_tlist == p->p_tlist->t_forw));
 
+	if (bp->b_data_size > 0) {
+		brand_data = kmem_zalloc(bp->b_data_size, KM_SLEEP);
+	}
+
+	mutex_enter(&p->p_lock);
+	ASSERT(!PROC_IS_BRANDED(p));
 	p->p_brand = bp;
+	p->p_brand_data = brand_data;
 	ASSERT(PROC_IS_BRANDED(p));
 	BROP(p)->b_setbrand(p);
+	mutex_exit(&p->p_lock);
 }
 
 void
 brand_clearbrand(proc_t *p)
 {
 	brand_t *bp = p->p_zone->zone_brand;
-	klwp_t *lwp = NULL;
-	ASSERT(bp != NULL);
-	ASSERT(PROC_IS_BRANDED(p));
-	VERIFY(mutex_owned(&p->p_lock));
+	void *brand_data;
+
+	VERIFY(MUTEX_NOT_HELD(&p->p_lock));
+	VERIFY(bp != NULL);
+	VERIFY(PROC_IS_BRANDED(p));
+
+	/*
+	 * There cannot be more than one lwp associated with a process when
+	 * stripping the brand.
+	 */
 	VERIFY((p->p_tlist == NULL) || (p->p_tlist == p->p_tlist->t_forw));
 
+	mutex_enter(&p->p_lock);
 	p->p_brand = &native_brand;
-	if (p->p_brand_data != NULL) {
-		kmem_free(p->p_brand_data, bp->b_data_size);
-		p->p_brand_data = NULL;
+	brand_data = p->p_brand_data;
+	p->p_brand_data = NULL;
+	mutex_exit(&p->p_lock);
+
+	if (brand_data != NULL) {
+		kmem_free(brand_data, bp->b_data_size);
 	}
 }
 
@@ -479,7 +498,7 @@ brand_solaris_cmd(int cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
 		return (ENOSYS);
 
 	/* For all other operations this must be a branded process. */
-	if (p->p_brand == &native_brand)
+	if (!PROC_IS_BRANDED(p))
 		return (ENOSYS);
 
 	ASSERT(p->p_brand == pbrand);
@@ -1018,7 +1037,7 @@ brand_solaris_exec(struct brand *pbrand)
 
 	/* Upon exec, reset our lwp brand data. */
 	(void) brand_solaris_freelwp(ttolwp(curthread), pbrand);
-	(void) brand_solaris_initlwp(ttolwp(curthread), pbrand);
+	(void) brand_solaris_brandlwp(ttolwp(curthread), pbrand);
 
 	/*
 	 * Upon exec, reset all the proc brand data, except for the elf
@@ -1062,7 +1081,7 @@ brand_solaris_forklwp(klwp_t *p, klwp_t *c, struct brand *pbrand)
 
 	/*
 	 * Both LWPs have already had been initialized via
-	 * brand_solaris_initlwp().
+	 * brand_solaris_brandlwp().
 	 */
 	ASSERT(p->lwp_brand != NULL);
 	ASSERT(c->lwp_brand != NULL);
@@ -1080,7 +1099,7 @@ brand_solaris_freelwp(klwp_t *l, struct brand *pbrand)
 
 /*ARGSUSED*/
 int
-brand_solaris_initlwp(klwp_t *l, struct brand *pbrand)
+brand_solaris_brandlwp(klwp_t *l, struct brand *pbrand)
 {
 	ASSERT(l->lwp_procp->p_brand == pbrand);
 	ASSERT(l->lwp_procp->p_brand_data != NULL);
@@ -1123,5 +1142,4 @@ brand_solaris_setbrand(proc_t *p, struct brand *pbrand)
 	ASSERT(p->p_tlist == p->p_tlist->t_forw);
 
 	p->p_brand_data = kmem_zalloc(sizeof (brand_proc_data_t), KM_SLEEP);
-	(void) brand_solaris_initlwp(p->p_tlist->t_lwp, pbrand);
 }