Simplify and document OpenZFS library dependencies

For those not already familiar with the code base it can be a
challenge to understand how the libraries are laid out.  This
has sometimes resulted in functionality being added in the
wrong place.  To help avoid that in the future this commit
documents the high-level dependencies for easy reference in
lib/Makefile.am.  It also simplifies a few things.

- Switched libzpool dependency on libzfs_core to libzutil.
  This change makes it clear libzpool should never depend
  on the ioctl() functionality provided by libzfs_core.

- Moved zfs_ioctl_fd() from libzutil to libzfs_core and
  renamed it lzc_ioctl_fd().  Normal access to the kmods
  should all be funneled through the libzfs_core library.
  The sole exception is the pool_active() which was updated
  to not use lzc_ioctl_fd() to remove the libzfs_core
  dependency.

- Removed libzfs_core dependency on libzutil.

- Removed the lib/libzfs/os/freebsd/libzfs_ioctl_compat.c
  source file which was all dead code.

- Removed libzfs_core dependency from mkbusy and ctime
  test utilities.  It was only needed for some trivial
  wrapper functions and that code is easy to replicate
  to shed the unneeded dependency.

Reviewed-by: Ryan Moeller <ryan@ixsystems.com>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Don Brady <don.brady@delphix.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes openzfs#12602

include/libzfs_core.h

@@ -41,6 +41,9 @@ extern "C" {
 int libzfs_core_init(void);
 void libzfs_core_fini(void);
 
+struct zfs_cmd;
+_LIBZFS_CORE_H int lzc_ioctl_fd(int, unsigned long, struct zfs_cmd *);
+
 /*
  * NB: this type should be kept binary compatible with dmu_objset_type_t.
  */

include/libzutil.h

@@ -145,7 +145,6 @@ extern int zpool_history_unpack(char *, uint64_t, uint64_t *, nvlist_t ***,
     uint_t *);
 
 struct zfs_cmd;
-int zfs_ioctl_fd(int fd, unsigned long request, struct zfs_cmd *zc);
 
 /*
  * List of colors to use

include/os/freebsd/zfs/sys/zfs_ioctl_compat.h

@@ -148,7 +148,6 @@ nvlist_t *zfs_ioctl_compat_outnvl(zfs_cmd_t *, nvlist_t *, const int,
 int zfs_ioctl_legacy_to_ozfs(int request);
 int zfs_ioctl_ozfs_to_legacy(int request);
 void zfs_cmd_legacy_to_ozfs(zfs_cmd_legacy_t *src, zfs_cmd_t *dst);
-void zfs_cmd_compat_get(zfs_cmd_t *, caddr_t, const int);
 void zfs_cmd_ozfs_to_legacy(zfs_cmd_t *src, zfs_cmd_legacy_t *dst);
 
 void zfs_cmd_compat_put(zfs_cmd_t *, caddr_t, const int, const int);

lib/Makefile.am

@@ -1,5 +1,44 @@
+#
+# Shown below is a simplified dependency graph of the OpenZFS provided
+# libraries.  Administrative commands (`zfs`, `zpool`, etc) interface with
+# the kernel modules using the `libzfs.so` and `libzfs_core.so` libraries.
+# These libraries provide a stable ABI across OpenZFS point releases.
+#
+# The `libzpool.so` library is a user space build of the DMU and SPA layers
+# used to implement debugging tools (zdb) and code validation tools (ztest).
+# These library interfaces are subject to change at any time.
+#
+#
+# CMDS:   zhack/ztest/zdb/            zfs/zpool/zed/
+#        raidz_{test,bench}          zinject/zstream
+#                |                          |
+# LIBS:          |                          |              libzfsbootenv*
+#                |                          |                    |
+#                |                          |                    |
+#             libzpool                   libzfs* ----------------+
+#             | | |  \                  / | | |
+#    libicp --/ | |   \                /  | | \------- libshare
+#               | |    \              /   | |
+#    libzstd ---/ |     \            /    | \--------- libuutil
+#                 |      \          /     \              | |
+#    libunicode --/       \        /       \             | |
+#                          \      /         \            | |
+#                          libzutil        libzfs_core*  | |
+#                          | | | | \          |     |    | |
+#                          | | | | |          |     |    | |
+#                          | | | | |          |     |    | |
+#    libtpool -------------/ | | | \---- libnvpair* |    | |
+#                            | | |                  |    | |
+#    libefi -----------------/ | \------ libavl* --------/ |
+#                              |                    |      |
+#                              \-------- libspl ----+------/
+#
+# * - A stable ABI is provided for these libraries
+#
+#
 # NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries
 # These nine libraries are intermediary build components.
+#
 SUBDIRS = libavl libicp libshare libspl libtpool libzstd
 CPPCHECKDIRS  = libavl libicp libnvpair libshare libspl libtpool libunicode
 CPPCHECKDIRS += libuutil libzfs libzfs_core libzfsbootenv libzpool libzutil

lib/libzfs/Makefile.am

@@ -33,7 +33,6 @@ USER_C = \
 if BUILD_FREEBSD
 USER_C += \
 	os/freebsd/libzfs_compat.c \
-	os/freebsd/libzfs_ioctl_compat.c \
 	os/freebsd/libzfs_zmount.c
 endif
 
@@ -73,6 +72,7 @@ libzfs_la_LIBADD = \
 	$(abs_top_builddir)/lib/libshare/libshare.la \
 	$(abs_top_builddir)/lib/libzfs_core/libzfs_core.la \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
+	$(abs_top_builddir)/lib/libzutil/libzutil.la \
 	$(abs_top_builddir)/lib/libuutil/libuutil.la
 
 libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LIBFETCH_LIBS) $(LTLIBINTL)

lib/libzfs/os/freebsd/libzfs_compat.c

@@ -22,7 +22,6 @@
 /*
  * Copyright (c) 2013 Martin Matuska <mm@FreeBSD.org>. All rights reserved.
  */
-#include <os/freebsd/zfs/sys/zfs_ioctl_compat.h>
 #include <libzfs_impl.h>
 #include <libzfs.h>
 #include <libzutil.h>
@@ -201,7 +200,7 @@ libzfs_error_init(int error)
 int
 zfs_ioctl(libzfs_handle_t *hdl, int request, zfs_cmd_t *zc)
 {
-	return (zfs_ioctl_fd(hdl->libzfs_fd, request, zc));
+	return (lzc_ioctl_fd(hdl->libzfs_fd, request, zc));
 }
 
 /*