replace fhs reuse with cache list (avoid ABA problem)

src/main/main.c

@@ -71,7 +71,7 @@ enum {
 /** File descriptor handler struct */
 struct fhs {
 	struct le he;        /**< Hash entry                        */
-	struct le le_delete; /**< Delete entry                      */
+	struct le le; 	     /**< Cache/Delete entry                */
 	int index;           /**< Index used for arrays             */
 	re_sock_t fd;        /**< File Descriptor                   */
 	int flags;           /**< Polling flags (Read, Write, etc.) */
@@ -83,7 +83,7 @@ struct fhs {
 struct re {
 	struct hash *fhl;            /**< File descriptor hash list         */
 	struct list fhs_delete;      /**< File descriptor delete list       */
-	bool fhs_reuse;              /**< Reuse file descriptors            */
+	struct list fhs_cache;       /**< File descriptor cache list        */
 	int maxfds;                  /**< Maximum number of polling fds     */
 	int max_fd;                  /**< Maximum fd number                 */
 	int nfds;                    /**< Number of active file descriptors */
@@ -127,6 +127,8 @@ static void re_destructor(void *arg)
 	mem_deref(re->mutex);
 	hash_flush(re->fhl);
 	mem_deref(re->fhl);
+	list_flush(&re->fhs_delete);
+	list_flush(&re->fhs_cache);
 }
 
 
@@ -163,12 +165,7 @@ int re_alloc(struct re **rep)
 
 	list_init(&re->tmrl);
 	list_init(&re->fhs_delete);
-
-#ifdef WIN32
-	re->fhs_reuse = false;
-#else
-	re->fhs_reuse = true;
-#endif
+	list_init(&re->fhs_cache);
 
 	re->tid = thrd_current();
 
@@ -633,6 +630,12 @@ static int fhs_update(struct re *re, struct fhs **fhsp, re_sock_t fd,
 	if (he) {
 		fhs = he->data;
 	}
+	else if (!list_isempty(&re->fhs_cache)) {
+		struct le *le = list_head(&re->fhs_cache);
+
+		fhs = le->data;
+		list_unlink(le);
+	}
 	else {
 		fhs = mem_zalloc(sizeof(struct fhs), fhs_destroy);
 		if (!fhs)
@@ -745,18 +748,12 @@ int fd_listen(re_sock_t fd, int flags, fd_h *fh, void *arg)
 #endif
 
 	if (!flags) {
-		if (!re->fhs_reuse) {
-			if (re->polling) {
-				list_append(&re->fhs_delete, &fhs->le_delete,
-					    fhs);
-				if (re->method == METHOD_EPOLL)
-					hash_unlink(&fhs->he);
-			}
-			else
-				mem_deref(fhs);
-
-		}
 		fhs->index = -1;
+		if (re_atomic_rlx(&re->polling))
+			list_append(&re->fhs_delete, &fhs->le, fhs);
+		else
+			list_append(&re->fhs_cache, &fhs->le, fhs);
+		hash_unlink(&fhs->he);
 		--re->nfds;
 	}
 
@@ -793,6 +790,7 @@ static int fd_poll(struct re *re)
 	const uint64_t to = tmr_next_timeout(&re->tmrl);
 	int n;
 	struct le *he;
+	struct le *le;
 	int nfds = re->nfds;
 #ifdef HAVE_SELECT
 	fd_set rfds, wfds, efds;
@@ -1027,7 +1025,12 @@ static int fd_poll(struct re *re)
 		--n;
 	}
 
-	list_flush(&re->fhs_delete);
+	LIST_FOREACH(&re->fhs_delete, le)
+	{
+		struct fhs *fhs = le->data;
+		list_unlink(le);
+		list_append(&re->fhs_cache, &fhs->le, fhs);
+	}
 
 	return 0;
 }
@@ -1443,9 +1446,6 @@ void re_thread_enter(void)
 
 	re_lock(re);
 
-	/* Disable fhs_reuse (no multithreading support) */
-	re->fhs_reuse = false;
-
 	/* set only for non-re threads */
 	if (!thrd_equal(re->tid, thrd_current())) {
 		re_atomic_rlx_set(&re->thread_enter, true);
@@ -1470,30 +1470,6 @@ void re_thread_leave(void)
 }
 
 
-/**
- * Set if the fhs allocation should be reused
- *
- * NOTE: POSIX defines that the lowest-numbered file descriptor is returned
- * for the current process. So re reuses the fhs hash entry by default
- * (Linux/Unix only). For lower memory usage this can be disabled.
- *
- * @param re     re context (NULL for re thread/global fallback)
- * @param reuse  If true reuse fhs hash entry, otherwise delete on fd_close
- */
-void re_fhs_reuse_set(struct re *re, bool reuse)
-{
-	if (!re)
-		re = re_get();
-
-	if (!re) {
-		DEBUG_WARNING("re_fhs_reuse_set: re not ready\n");
-		return;
-	}
-
-	re->fhs_reuse = reuse;
-}
-
-
 /**
  * Attach the current thread to re context
  */