plan9port

fork of plan9port with libvec, libstr and libsdb
Log | Files | Refs | README | LICENSE

commit baef953da253314657be9adea8f371bfbf4ba09e
parent 06687f70ba7a5836c2e872648a85a724a5a1d486
Author: Russ Cox <rsc@swtch.com>
Date:   Sun, 17 May 2020 08:24:54 -0400

libthread: add pthreadperthread mode and use under ASAN

ASAN can't deal with the coroutine stacks.
In theory we can call into ASAN runtime to let it know about them,
but ASAN still has problems with fork or exit happening from a
non-system stack. Bypass all possible problems by just having
a full OS thread for each libthread thread. The threads are still
cooperatively scheduled within a proc (in thos mode, a group of OS threads).

Setting the environment variable LIBTHREAD=pthreadperthread
will enable the pthreadperthread mode, as will building with
CC9FLAGS='-fsanitize=address' in $PLAN9/config.

This solution is much more general than ASAN - for example if
you are trying to find all the thread stacks in a reproducible crash
you can use pthreadperthread mode with any debugger that
knows only about OS threads.

Diffstat:
Msrc/libthread/channel.c | 2+-
Msrc/libthread/pthread.c | 33+++++++++++++++++++++++++++++++++
Msrc/libthread/thread.c | 70++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
Msrc/libthread/threadimpl.h | 33++++++++++++++++++++++-----------
4 files changed, 120 insertions(+), 18 deletions(-)

diff --git a/src/libthread/channel.c b/src/libthread/channel.c @@ -141,7 +141,7 @@ altdequeue(Alt *a) delarray(ar, i); return; } - fprint(2, "cannot find self in altdq\n"); + fprint(2, "cannot find self in altdequeue\n"); abort(); } diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c @@ -99,6 +99,23 @@ startprocfn(void *v) pthread_exit(0); } +static void +startpthreadfn(void *v) +{ + void **a; + Proc *p; + _Thread *t; + + a = (void**)v; + p = a[0]; + t = a[1]; + free(a); + t->osprocid = pthread_self(); + pthread_detach(t->osprocid); + _threadpthreadmain(p, t); + pthread_exit(0); +} + void _procstart(Proc *p, void (*fn)(Proc*)) { @@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*)) } } +void +_threadpthreadstart(Proc *p, _Thread *t) +{ + void **a; + + a = malloc(3*sizeof a[0]); + if(a == nil) + sysfatal("_pthreadstart malloc: %r"); + a[0] = p; + a[1] = t; + if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){ + fprint(2, "pthread_create: %r\n"); + abort(); + } +} + static pthread_key_t prockey; Proc* diff --git a/src/libthread/thread.c b/src/libthread/thread.c @@ -7,6 +7,7 @@ static uint threadnsysproc; static Lock threadnproclock; static Ref threadidref; static Proc *threadmainproc; +static int pthreadperthread; static void addproc(Proc*); static void delproc(Proc*); @@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack) if(stack < (256<<10)) stack = 256<<10; - if(p->nthread == 0) + if(p->nthread == 0 || pthreadperthread) stack = 0; // not using it t = threadalloc(fn, arg, stack); t->proc = p; - addthreadinproc(p, t); + if(p->nthread == 0) + p->thread0 = t; + else if(pthreadperthread) + _threadpthreadstart(p, t); p->nthread++; + addthreadinproc(p, t); _threadready(t); return t; } @@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack) return id; } +// For pthreadperthread mode, procswitch flips +// between the threads. +static void +procswitch(Proc *p, _Thread *from, _Thread *to) +{ +// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1); + lock(&p->schedlock); + from->schedrend.l = &p->schedlock; + if(to) { + p->schedthread = to; + to->schedrend.l = &p->schedlock; + _procwakeup(&to->schedrend); + } + if(p->schedthread != from) { + if(from->exiting) { + unlock(&p->schedlock); + _threadpexit(); + } + _procsleep(&from->schedrend); + } + unlock(&p->schedlock); +} + void _threadswitch(void) { @@ -216,9 +244,13 @@ _threadswitch(void) needstack(0); p = proc(); + /*print("threadswtch %p\n", p); */ - if(p->thread->stk == nil) + + if(p->thread == p->thread0) procscheduler(p); + else if(pthreadperthread) + procswitch(p, p->thread, p->thread0); else contextswitch(&p->thread->context, &p->schedcontext); } @@ -346,6 +378,15 @@ procmain(Proc *p) threadexits(nil); } +void +_threadpthreadmain(Proc *p, _Thread *t) +{ + _threadsetproc(p); + procswitch(p, t, nil); + t->startfn(t->startarg); + threadexits(nil); +} + static void procscheduler(Proc *p) { @@ -401,9 +442,12 @@ Top: p->nswitch++; _threaddebug("run %d (%s)", t->id, t->name); //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si); - if(t->stk == nil) + if(t == p->thread0) return; - contextswitch(&p->schedcontext, &t->context); + if(pthreadperthread) + procswitch(p, p->thread0, t); + else + contextswitch(&p->schedcontext, &t->context); /*print("back in scheduler\n"); */ goto Top; } @@ -757,10 +801,24 @@ int main(int argc, char **argv) { Proc *p; + char *opts; argv0 = argv[0]; - if(getenv("NOLIBTHREADDAEMONIZE") == nil) + opts = getenv("LIBTHREAD"); + if(opts == nil) + opts = ""; + + pthreadperthread = (strstr(opts, "pthreadperthread") != nil); +#ifdef PLAN9PORT_ASAN + // ASAN can't deal with the coroutine stack switches. + // In theory it has support for informing it about stack switches, + // but even with those calls added it can't deal with things + // like fork or exit from a coroutine stack. + // Easier to just run in pthread-per-thread mode. + pthreadperthread = 1; +#endif + if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil) _threadsetupdaemonize(); threadargc = argc; diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h @@ -102,6 +102,17 @@ struct Execjob Channel *c; }; +struct _Procrendez +{ + Lock *l; + int asleep; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_cond_t cond; +#else + int pid; +#endif +}; + struct _Thread { _Thread *next; @@ -112,6 +123,11 @@ struct _Thread void (*startfn)(void*); void *startarg; uint id; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_t osprocid; +#else + int osprocid; +#endif uchar *stk; uint stksize; int exiting; @@ -120,17 +136,7 @@ struct _Thread char state[256]; void *udata; Alt *alt; -}; - -struct _Procrendez -{ - Lock *l; - int asleep; -#ifdef PLAN9PORT_USING_PTHREADS - pthread_cond_t cond; -#else - int pid; -#endif + _Procrendez schedrend; }; extern void _procsleep(_Procrendez*); @@ -149,6 +155,7 @@ struct Proc #endif Lock lock; int nswitch; + _Thread *thread0; _Thread *thread; _Thread *pinthread; _Threadlist runqueue; @@ -157,6 +164,8 @@ struct Proc uint nthread; uint sysproc; _Procrendez runrend; + Lock schedlock; + _Thread *schedthread; Context schedcontext; void *udata; Jmp sigjmp; @@ -188,6 +197,8 @@ extern void _threadpexit(void); extern void _threaddaemonize(void); extern void *_threadstkalloc(int); extern void _threadstkfree(void*, int); +extern void _threadpthreadmain(Proc*, _Thread*); +extern void _threadpthreadstart(Proc*, _Thread*); #define USPALIGN(ucp, align) \ (void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1))