Support -fsanitize=address with --enable-asan

cmd/raidz_test/Makefile.am

@@ -1,7 +1,10 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
-AM_CPPFLAGS += -DDEBUG
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
+
+# Unconditionally enable ASSERTs
+AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \

cmd/zdb/Makefile.am

@@ -1,6 +1,7 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CPPFLAGS += -DDEBUG
+# Unconditionally enable debugging for zdb
+AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \

cmd/ztest/Makefile.am

@@ -1,8 +1,12 @@
 include $(top_srcdir)/config/Rules.am
 
-# -Wnoformat-truncation to get rid of compiler warning for unchecked
-#  truncating snprintfs on gcc 7.1.1.
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN) $(NO_FORMAT_TRUNCATION)
+# Get rid of compiler warning for unchecked truncating snprintfs on gcc 7.1.1
+AM_CFLAGS += $(NO_FORMAT_TRUNCATION)
+
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
+
+# Unconditionally enable ASSERTs
 AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \

config/Rules.am

@@ -1,18 +1,29 @@
+#
+# Default build rules for all user space components, every Makefile.am
+# should include these rules and override or extend them as needed.
+#
+
 DEFAULT_INCLUDES = -include ${top_builddir}/zfs_config.h
 
 AM_LIBTOOLFLAGS = --silent
-AM_CFLAGS  = ${DEBUG_CFLAGS} -Wall -Wstrict-prototypes
-AM_CFLAGS += ${NO_UNUSED_BUT_SET_VARIABLE}
-AM_CFLAGS += ${NO_BOOL_COMPARE}
-AM_CFLAGS += -fno-strict-aliasing
-AM_CFLAGS += -std=gnu99
+
+AM_CFLAGS  = -std=gnu99 -Wall -Wstrict-prototypes -fno-strict-aliasing
+AM_CFLAGS += $(DEBUG_CFLAGS)
+AM_CFLAGS += $(ASAN_CFLAGS)
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
-AM_CPPFLAGS  = -D_GNU_SOURCE -D__EXTENSIONS__ -D_REENTRANT
-AM_CPPFLAGS += -D_POSIX_PTHREAD_SEMANTICS -D_FILE_OFFSET_BITS=64
-AM_CPPFLAGS += -D_LARGEFILE64_SOURCE -DHAVE_LARGE_STACKS=1
+
+AM_CPPFLAGS  = -D_GNU_SOURCE
+AM_CPPFLAGS += -D_REENTRANT
+AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64
+AM_CPPFLAGS += -D_LARGEFILE64_SOURCE
+AM_CPPFLAGS += -DHAVE_LARGE_STACKS=1
 AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
 AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
 AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
 AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
 AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
+AM_CPPFLAGS += $(DEBUG_CPPFLAGS)
 AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS)
+
+AM_LDFLAGS  = $(DEBUG_LDFLAGS)
+AM_LDFLAGS += $(ASAN_LDFLAGS)

config/always-compiler-options.m4

@@ -0,0 +1,141 @@
+dnl #
+dnl # Enabled -fsanitize=address if supported by gcc.
+dnl #
+dnl # LDFLAGS needs -fsanitize=address at all times so libraries compiled with
+dnl # it will be linked successfully. CFLAGS will vary by binary being built.
+dnl #
+dnl # The ASAN_OPTIONS environment variable can be used to further control
+dnl # the behavior of binaries and libraries build with -fsanitize=address.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_ASAN], [
+	AC_MSG_CHECKING([whether to build with -fsanitize=address support])
+	AC_ARG_ENABLE([asan],
+		[AS_HELP_STRING([--enable-asan],
+		[Enable -fsanitize=address support  @<:@default=no@:>@])],
+		[],
+		[enable_asan=no])
+
+	AM_CONDITIONAL([ASAN_ENABLED], [test x$enable_asan = xyes])
+	AC_SUBST([ASAN_ENABLED], [$enable_asan])
+	AC_MSG_RESULT($enable_asan)
+
+	AS_IF([ test "$enable_asan" = "yes" ], [
+		AC_MSG_CHECKING([whether $CC supports -fsanitize=address])
+		saved_cflags="$CFLAGS"
+		CFLAGS="$CFLAGS -fsanitize=address"
+		AC_LINK_IFELSE([
+			AC_LANG_SOURCE([[ int main() { return 0; } ]])
+		], [
+			ASAN_CFLAGS="-fsanitize=address"
+			ASAN_LDFLAGS="-fsanitize=address"
+			ASAN_ZFS="_with_asan"
+			AC_MSG_RESULT([yes])
+		], [
+			AC_MSG_ERROR([$CC does not support -fsanitize=address])
+		])
+		CFLAGS="$saved_cflags"
+	], [
+		ASAN_CFLAGS=""
+		ASAN_LDFLAGS=""
+		ASAN_ZFS="_without_asan"
+	])
+
+	AC_SUBST([ASAN_CFLAGS])
+	AC_SUBST([ASAN_LDFLAGS])
+	AC_SUBST([ASAN_ZFS])
+])
+
+dnl #
+dnl # Check if gcc supports -Wframe-larger-than=<size> option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN], [
+	AC_MSG_CHECKING([whether $CC supports -Wframe-larger-than=<size>])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wframe-larger-than=4096"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		FRAME_LARGER_THAN="-Wframe-larger-than=4096"
+		AC_MSG_RESULT([yes])
+	], [
+		FRAME_LARGER_THAN=""
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([FRAME_LARGER_THAN])
+])
+
+dnl #
+dnl # Check if gcc supports -Wno-format-truncation option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-format-truncation])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wno-format-truncation"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_FORMAT_TRUNCATION=-Wno-format-truncation
+		AC_MSG_RESULT([yes])
+	], [
+		NO_FORMAT_TRUNCATION=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_FORMAT_TRUNCATION])
+])
+
+
+dnl #
+dnl # Check if gcc supports -Wno-bool-compare option.
+dnl #
+dnl # We actually invoke gcc with the -Wbool-compare option
+dnl # and infer the 'no-' version does or doesn't exist based upon
+dnl # the results.  This is required because when checking any of
+dnl # no- prefixed options gcc always returns success.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-bool-compare])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wbool-compare"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_BOOL_COMPARE=-Wno-bool-compare
+		AC_MSG_RESULT([yes])
+	], [
+		NO_BOOL_COMPARE=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_BOOL_COMPARE])
+])
+
+dnl #
+dnl # Check if gcc supports -Wno-unused-but-set-variable option.
+dnl #
+dnl # We actually invoke gcc with the -Wunused-but-set-variable option
+dnl # and infer the 'no-' version does or doesn't exist based upon
+dnl # the results.  This is required because when checking any of
+dnl # no- prefixed options gcc always returns success.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-unused-but-set-variable])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wunused-but-set-variable"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable
+		AC_MSG_RESULT([yes])
+	], [
+		NO_UNUSED_BUT_SET_VARIABLE=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_UNUSED_BUT_SET_VARIABLE])
+])

config/kernel.m4

@@ -125,21 +125,10 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
 	ZFS_AC_KERNEL_VM_NODE_STAT
 
 	AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
-		KERNELMAKE_PARAMS="$KERNELMAKE_PARAMS O=$LINUX_OBJ"
+		KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
 	])
-	AC_SUBST(KERNELMAKE_PARAMS)
 
-
-	dnl # -Wall -fno-strict-aliasing -Wstrict-prototypes and other
-	dnl # compiler options are added by the kernel build system.
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -std=gnu99"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -Wno-declaration-after-statement"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS $NO_UNUSED_BUT_SET_VARIABLE"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS $NO_BOOL_COMPARE"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -DHAVE_SPL -D_KERNEL"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -DTEXT_DOMAIN=\\\"zfs-linux-kernel\\\""
-
-	AC_SUBST(KERNELCPPFLAGS)
+	AC_SUBST(KERNEL_MAKE)
 ])
 
 dnl #

config/user.m4

@@ -14,11 +14,9 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBATTR
 	ZFS_AC_CONFIG_USER_LIBUDEV
 	ZFS_AC_CONFIG_USER_LIBSSL
-	ZFS_AC_CONFIG_USER_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_MKDEV
-	ZFS_AC_CONFIG_USER_NO_FORMAT_TRUNCATION
 
 	ZFS_AC_TEST_FRAMEWORK