Atomically initialize parts of the 2LS (XCC)

The issue is that certain parts of the 2LS must be changed atomically
with respect to syscalls, meaning no syscalls happen between the
operations.

For instance, if you change the syscall event handler, you need to switch
over to using the correct 2LS ops first.  Otherwise, the handler will go
crazy.

Likewise, we need to register for INDIRs, in case the 2LS uses INDIRs for
its syscall handling (all of them do).  Otherwise, you'd get the INDIR, but
that would get ignored (since there's no handler).

Finally, this fully removes thread0's syscall handler.  We actually had
always been running thread0's handler - the event handers chain.  It just
so happened to not be harmful.

Rebuild all apps and libraries, though maybe not glibc.  Specifically, I
needed to rebuild elfutils, since libelf.so has copies of parlib stuff in
it.

Signed-off-by: Barret Rhoden <brho@cs.berkeley.edu>

diff --git a/tests/mcp_halt.c b/tests/mcp_halt.c
index 4231581..d7004d9 100644 (file)
--- a/tests/mcp_halt.c
+++ b/tests/mcp_halt.c
@@ -40,7 +40,7 @@ int main(int argc, char** argv)
        /* Inits a thread for us, though we won't use it.  Just a hack to get into
         * _M mode.  Note this requests one vcore for us */
        struct uthread dummy = {0};
-       uthread_2ls_init(&dummy, &ghetto_sched_ops);
+       uthread_2ls_init(&dummy, &ghetto_sched_ops, NULL, NULL);
        uthread_mcp_init();
 
        /* Reset the blockon to be the spinner...  This is really shitty.  Any

diff --git a/tests/mhello.c b/tests/mhello.c
index ffbd2d0..5416dbb 100644 (file)
--- a/tests/mhello.c
+++ b/tests/mhello.c
@@ -72,7 +72,7 @@ int main(int argc, char** argv)
        /* Inits a thread for us, though we won't use it.  Just a hack to get into
         * _M mode.  Note this requests one vcore for us */
        struct uthread dummy = {0};
-       uthread_2ls_init(&dummy, &ghetto_sched_ops);
+       uthread_2ls_init(&dummy, &ghetto_sched_ops, NULL, NULL);
        uthread_mcp_init();
        /* Reset the blockon to be the spinner...  This is really shitty.  Any
         * blocking calls after we become an MCP and before this will fail.  This is

diff --git a/user/parlib/event.c b/user/parlib/event.c
index 46b9e26..3a6f7a7 100644 (file)
--- a/user/parlib/event.c
+++ b/user/parlib/event.c
@@ -271,7 +271,9 @@ spinpdrlock_t ev_h_wlock = SPINPDR_INITIALIZER;
 int register_ev_handler(unsigned int ev_type, handle_event_t handler,
                         void *data)
 {
+       /* Nasty uthread code assumes this was malloced */
        struct ev_handler *new_h = malloc(sizeof(struct ev_handler));
+
        if (!new_h)
                return -1;
        new_h->func = handler;

diff --git a/user/parlib/include/parlib/event.h b/user/parlib/include/parlib/event.h
index a5fe360..d93f680 100644 (file)
--- a/user/parlib/include/parlib/event.h
+++ b/user/parlib/include/parlib/event.h
@@ -47,6 +47,7 @@ struct ev_handler {
        handle_event_t                          func;
        void                                            *data;
 };
+extern struct ev_handler *ev_handlers[];
 int register_ev_handler(unsigned int ev_type, handle_event_t handler,
                         void *data);
 int deregister_ev_handler(unsigned int ev_type, handle_event_t handler,

diff --git a/user/parlib/include/parlib/uthread.h b/user/parlib/include/parlib/uthread.h
index 3390a88..5cf1b06 100644 (file)
--- a/user/parlib/include/parlib/uthread.h
+++ b/user/parlib/include/parlib/uthread.h
@@ -74,9 +74,14 @@ extern struct schedule_ops *sched_ops;
 
 /* Low-level _S code calls this for basic uthreading without a 2LS */
 void uthread_lib_init(void);
-/* Call this, passing it a uthread representing thread0, from your 2LS init
- * routines.  When it returns, you're in _M mode (thread0 on vcore0) */
-void uthread_2ls_init(struct uthread *uthread, struct schedule_ops *ops);
+/* Call this from your 2LS init routines.  Pass it a uthread representing
+ * thread0, your 2LS ops, and your syscall handler + data.
+ *
+ * When it returns, you're in _M mode (thread0 on vcore0) */
+void uthread_2ls_init(struct uthread *uthread, struct schedule_ops *ops,
+                      void (*handle_sysc)(struct event_msg *, unsigned int,
+                                          void *),
+                      void *data);
 /* Call this to become an mcp capable of worling with uthreads. */
 void uthread_mcp_init(void);
 

diff --git a/user/parlib/thread0_sched.c b/user/parlib/thread0_sched.c
index 7571a50..b5b2d84 100644 (file)
--- a/user/parlib/thread0_sched.c
+++ b/user/parlib/thread0_sched.c
@@ -53,8 +53,8 @@ struct thread0_info {
 static struct thread0_info thread0_info;
 static struct event_queue *sysc_evq;
 
-static void thread0_handle_syscall(struct event_msg *ev_msg,
-                                   unsigned int ev_type, void *data)
+void thread0_handle_syscall(struct event_msg *ev_msg,
+                            unsigned int ev_type, void *data)
 {
        thread0_info.is_blocked = FALSE;
 }
@@ -65,7 +65,6 @@ void thread0_lib_init(void)
        /* we don't care about the message, so don't bother with a UCQ */
        sysc_evq = get_eventq(EV_MBOX_BITMAP);
        sysc_evq->ev_flags = EVENT_INDIR | EVENT_WAKEUP;
-       register_ev_handler(EV_SYSCALL, thread0_handle_syscall, 0);
 }
 
 /* Thread0 scheduler ops (for processes that haven't linked in a full 2LS) */

diff --git a/user/parlib/uthread.c b/user/parlib/uthread.c
index 1cacdee..c9a706b 100644 (file)
--- a/user/parlib/uthread.c
+++ b/user/parlib/uthread.c
@@ -134,9 +134,28 @@ void uthread_mcp_init()
        vcore_change_to_m();
 }
 
-/* The real 2LS calls this, passing in a uthread representing thread0. */
-void uthread_2ls_init(struct uthread *uthread, struct schedule_ops *ops)
+/* The real 2LS calls this, passing in a uthread representing thread0, its 2LS
+ * ops, and its syscall handling routine.  (NULL is fine).
+ *
+ * When we're called, thread0 has it's handler installed.  We need to remove it
+ * and install our own (if they want one).  All of the actual changes  must be
+ * done atomically with respect to syscalls (i.e., no syscalls in between).  If
+ * the user did something like have an outstanding async syscall while we're in
+ * here, then they'll crash hard. */
+void uthread_2ls_init(struct uthread *uthread, struct schedule_ops *ops,
+                      void (*handle_sysc)(struct event_msg *, unsigned int,
+                                          void *),
+                      void *data)
 {
+       struct ev_handler *old_h, *new_h = NULL;
+
+       if (handle_sysc) {
+               new_h = malloc(sizeof(struct ev_handler));
+               assert(new_h);
+               new_h->func = handle_sysc;
+               new_h->data = data;
+               new_h->next = NULL;
+       }
        uthread_init_thread0(uthread);
        /* We need to *atomically* change the current_uthread and the schedule_ops
         * to the new 2LSs thread0 and ops, such that there is no moment when only
@@ -158,9 +177,13 @@ void uthread_2ls_init(struct uthread *uthread, struct schedule_ops *ops)
         * previously). */
        uthread_track_thread0(uthread);
        sched_ops = ops;
+       /* Racy, but we shouldn't be concurrent */
+       old_h = ev_handlers[EV_SYSCALL];
+       ev_handlers[EV_SYSCALL] = new_h;
        cmb();
        __vcore_context = FALSE;
        enable_notifs(0);       /* will trigger a self_notif if we missed a notif */
+       free(old_h);
 }
 
 /* Helper: tells the kernel our SCP is capable of going into vcore context on
@@ -207,6 +230,8 @@ void __attribute__((constructor)) uthread_lib_init(void)
        /* Use the thread0 sched's uth */
        extern struct uthread *thread0_uth;
        extern void thread0_lib_init(void);
+       extern void thread0_handle_syscall(struct event_msg *ev_msg,
+                                          unsigned int ev_type, void *data);
        int ret;
 
        /* Only run once, but make sure that vcore_lib_init() has run already. */
@@ -217,9 +242,20 @@ void __attribute__((constructor)) uthread_lib_init(void)
                             sizeof(struct uthread));
        assert(!ret);
        memset(thread0_uth, 0, sizeof(struct uthread)); /* aggressively 0 for bugs*/
-       /* Init the 2LS, which sets up current_uthread, before thread0 lib */
-       uthread_2ls_init(thread0_uth, &thread0_2ls_ops);
+       /* Need to do thread0_lib_init() first, since it sets up evq's referred to
+        * in thread0_handle_syscall(). */
        thread0_lib_init();
+       uthread_2ls_init(thread0_uth, &thread0_2ls_ops, thread0_handle_syscall,
+                        NULL);
+       /* Switch our errno/errstr functions to be uthread-aware.  See glibc's
+        * errno.c for more info. */
+       ros_errno_loc = __ros_errno_loc;
+       ros_errstr_loc = __ros_errstr_loc;
+       register_ev_handler(EV_EVENT, handle_ev_ev, 0);
+       /* Now that we're ready (I hope) to operate as an MCP, we tell the kernel.
+        * We must set vcctx and blockon atomically with respect to syscalls,
+        * meaning no syscalls in between. */
+       cmb();
        scp_vcctx_ready();
        /* Change our blockon from glibc's internal one to the regular one, which
         * uses vcore context and works for SCPs (with or without 2LS) and MCPs.
@@ -228,11 +264,6 @@ void __attribute__((constructor)) uthread_lib_init(void)
        ros_syscall_blockon = __ros_uth_syscall_blockon;
        cmb();
        init_posix_signals();
-       /* Switch our errno/errstr functions to be uthread-aware.  See glibc's
-        * errno.c for more info. */
-       ros_errno_loc = __ros_errno_loc;
-       ros_errstr_loc = __ros_errstr_loc;
-       register_ev_handler(EV_EVENT, handle_ev_ev, 0);
        /* Accept diagnostic events.  Other parts of the program/libraries can
         * register handlers to run.  You can kick these with "notify PID 9". */
        enable_kevent(EV_FREE_APPLE_PIE, 0, EVENT_IPI | EVENT_WAKEUP |

diff --git a/user/pthread/pthread.c b/user/pthread/pthread.c
index 59faf38..5a3213b 100644 (file)
--- a/user/pthread/pthread.c
+++ b/user/pthread/pthread.c
@@ -471,9 +471,6 @@ void __attribute__((constructor)) pthread_lib_init(void)
         * private preference.  Also note that enable_kevent() is just an example,
         * and you probably want to use parts of event.c to do what you want. */
        enable_kevent(EV_USER_IPI, 0, EVENT_IPI | EVENT_VCORE_PRIVATE);
-
-       /* Handle syscall events. */
-       register_ev_handler(EV_SYSCALL, pth_handle_syscall, 0);
        /* Set up the per-vcore structs to track outstanding syscalls */
        sysc_mgmt = malloc(sizeof(struct sysc_mgmt) * max_vcores());
        assert(sysc_mgmt);
@@ -518,7 +515,8 @@ void __attribute__((constructor)) pthread_lib_init(void)
        }
 #endif
        /* Sched ops is set by 2ls_init */
-       uthread_2ls_init((struct uthread*)t, &pthread_sched_ops);
+       uthread_2ls_init((struct uthread*)t, &pthread_sched_ops, pth_handle_syscall,
+                        NULL);
        atomic_init(&threads_total, 1);                 /* one for thread0 */
 }
 

diff --git a/user/vmm/sched.c b/user/vmm/sched.c
index eb31ddc..ad28499 100644 (file)
--- a/user/vmm/sched.c
+++ b/user/vmm/sched.c
@@ -121,8 +121,8 @@ static void __attribute__((constructor)) vmm_lib_init(void)
        thread0->stacktop = (void*)USTACKTOP;
        /* for lack of a better vcore, might as well send to 0 */
        sysc_evq = setup_sysc_evq(0);
-       register_ev_handler(EV_SYSCALL, vmm_handle_syscall, 0);
-       uthread_2ls_init((struct uthread*)thread0, &vmm_sched_ops);
+       uthread_2ls_init((struct uthread*)thread0, &vmm_sched_ops,
+                     vmm_handle_syscall, NULL);
 }
 
 /* The scheduling policy is encapsulated in the next few functions (from here