replace pthreads cancellation with explicit checks

Due to data/event loss, we cannot rely on normal syscalls
(accept/epoll_wait) being cancellation points.  The benefits of
using a standardized API to terminate threads asynchronously are
lost when toggling cancellation flags.

This implementation allows us to be more explicit and obvious at the
few points where our worker threads may exit and reduces the amount
of code we have.  By avoiding the calls to pthread_setcancelstate,
we should halve the number of atomic operations required in the
common case (where the thread is not marked for termination).

1 files changed, 7 insertions, 2 deletions

diff --git a/cmogstored.h b/cmogstored.h
index a91f393..7b1f164 100644
--- a/cmogstored.h
+++ b/cmogstored.h
@@ -207,11 +207,16 @@ struct mog_http {
         struct mog_packaddr mpa;
 } __attribute__((packed));
 
+struct mog_thread {
+        pthread_t thr;
+        unsigned *do_quit;
+};
+
 struct mog_thrpool {
         pthread_mutex_t lock;
         size_t n_threads;
         size_t want_threads;
-        pthread_t *threads;
+        struct mog_thread *threads;
         void *(*start_fn)(void *);
         void *start_arg;
 };
@@ -262,7 +267,6 @@ struct mog_file {
 
 /* sig.c */
 extern sigset_t mog_emptyset;
-void mog_cancel_prepare(void);
 void mog_intr_disable(void);
 void mog_intr_enable(void);
 void mog_sleep(long seconds);
@@ -400,6 +404,7 @@ char *mog_canonpath(const char *path, enum canonicalize_mode_t canon_mode);
 char *mog_canonpath_die(const char *path, enum canonicalize_mode_t canon_mode);
 
 /* thrpool.c */
+void mog_thr_test_quit(void);
 void mog_thrpool_start(struct mog_thrpool *, size_t n,
                        void *(*start_fn)(void *), void *arg);
 void mog_thrpool_quit(struct mog_thrpool *, struct mog_queue *);