Don't allow shrinking a PF_FSTRANS context

Avoid deadlocks when entering the shrinker from a PF_FSTRANS context.

This patch also reverts commit d0d5dd7 which added MUTEX_FSTRANS.  Its
use has been deprecated within ZFS as it was an ineffective mechanism
to eliminate deadlocks.  Among other things, it introduced the need for
strict ordering of mutex locking and unlocking in order that the
PF_FSTRANS flag wouldn't set incorrectly.

Signed-off-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #446

include/sys/mutex.h

@@ -32,19 +32,29 @@
 typedef enum {
 	MUTEX_DEFAULT	= 0,
 	MUTEX_SPIN	= 1,
-	MUTEX_ADAPTIVE	= 2,
-	MUTEX_FSTRANS	= 3,
+	MUTEX_ADAPTIVE	= 2
 } kmutex_type_t;
 
 typedef struct {
 	struct mutex		m_mutex;
-	kmutex_type_t		m_type;
 	spinlock_t		m_lock;	/* used for serializing mutex_exit */
 	kthread_t		*m_owner;
-	unsigned int		m_saved_flags;
 } kmutex_t;
 
 #define	MUTEX(mp)		(&((mp)->m_mutex))
+
+static inline void
+spl_mutex_set_owner(kmutex_t *mp)
+{
+	mp->m_owner = current;
+}
+
+static inline void
+spl_mutex_clear_owner(kmutex_t *mp)
+{
+	mp->m_owner = NULL;
+}
+
 #define	mutex_owner(mp)		(ACCESS_ONCE((mp)->m_owner))
 #define	mutex_owned(mp)		(mutex_owner(mp) == current)
 #define	MUTEX_HELD(mp)		mutex_owned(mp)
@@ -60,18 +70,11 @@ typedef struct {
 #define	mutex_init(mp, name, type, ibc)				\
 {								\
 	static struct lock_class_key __key;			\
-								\
-	ASSERT3P(mp, !=, NULL);					\
-	ASSERT3P(ibc, ==, NULL);				\
-	ASSERT((type == MUTEX_DEFAULT) ||			\
-	    (type == MUTEX_ADAPTIVE) ||				\
-	    (type == MUTEX_FSTRANS));				\
+	ASSERT(type == MUTEX_DEFAULT);				\
 								\
 	__mutex_init(MUTEX(mp), (name) ? (#name) : (#mp), &__key); \
 	spin_lock_init(&(mp)->m_lock);				\
-	(mp)->m_type = type;					\
-	(mp)->m_owner = NULL;					\
-	(mp)->m_saved_flags = 0;				\
+	spl_mutex_clear_owner(mp);				\
 }
 
 #undef mutex_destroy
@@ -84,13 +87,8 @@ typedef struct {
 ({								\
 	int _rc_;						\
 								\
-	if ((_rc_ = mutex_trylock(MUTEX(mp))) == 1) {		\
-		(mp)->m_owner = current;			\
-		if ((mp)->m_type == MUTEX_FSTRANS) {		\
-			(mp)->m_saved_flags = current->flags;	\
-			current->flags |= PF_FSTRANS;		\
-		}						\
-	}							\
+	if ((_rc_ = mutex_trylock(MUTEX(mp))) == 1)		\
+		spl_mutex_set_owner(mp);			\
 								\
 	_rc_;							\
 })
@@ -100,22 +98,14 @@ typedef struct {
 {								\
 	ASSERT3P(mutex_owner(mp), !=, current);			\
 	mutex_lock_nested(MUTEX(mp), (subclass));		\
-	(mp)->m_owner = current;				\
-	if ((mp)->m_type == MUTEX_FSTRANS) {			\
-		(mp)->m_saved_flags = current->flags;		\
-		current->flags |= PF_FSTRANS;			\
-	}							\
+	spl_mutex_set_owner(mp);				\
 }
 #else /* CONFIG_DEBUG_LOCK_ALLOC */
 #define	mutex_enter_nested(mp, subclass)			\
 {								\
 	ASSERT3P(mutex_owner(mp), !=, current);			\
 	mutex_lock(MUTEX(mp));					\
-	(mp)->m_owner = current;				\
-	if ((mp)->m_type == MUTEX_FSTRANS) {			\
-		(mp)->m_saved_flags = current->flags;		\
-		current->flags |= PF_FSTRANS;			\
-	}							\
+	spl_mutex_set_owner(mp);				\
 }
 #endif /*  CONFIG_DEBUG_LOCK_ALLOC */
 
@@ -143,11 +133,7 @@ typedef struct {
 #define	mutex_exit(mp)						\
 {								\
 	spin_lock(&(mp)->m_lock);				\
-	if ((mp)->m_type == MUTEX_FSTRANS) {			\
-		current->flags &= ~(PF_FSTRANS);		\
-		current->flags |= (mp)->m_saved_flags;		\
-	}							\
-	(mp)->m_owner = NULL;					\
+	spl_mutex_clear_owner(mp);				\
 	mutex_unlock(MUTEX(mp));				\
 	spin_unlock(&(mp)->m_lock);				\
 }

module/spl/spl-kmem-cache.c

@@ -1574,6 +1574,12 @@ __spl_kmem_cache_generic_shrinker(struct shrinker *shrink,
 	spl_kmem_cache_t *skc;
 	int alloc = 0;
 
+	/*
+	 * No shrinking in a transaction context.  Can cause deadlocks.
+	 */
+	if (sc->nr_to_scan && spl_fstrans_check())
+		return (SHRINK_STOP);
+
 	down_read(&spl_kmem_cache_sem);
 	list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) {
 		if (sc->nr_to_scan) {