Cleanup linux module kbuild files

The linux module can be built either as an external module, or compiled
into the kernel, using copy-builtin. The source and build directories
are slightly different between the two cases, and currently, compiling
into the kernel still refers to some files from the configured ZFS
source tree, instead of the copies inside the kernel source tree. There
is also duplication between copy-builtin, which creates a Kbuild file to
build ZFS inside the kernel tree, and the top-level module/Makefile.in.

Fix this by moving the list of modules and the CFLAGS settings into a
new module/Kbuild.in, which will be used by the kernel kbuild
infrastructure, and using KBUILD_EXTMOD to distinguish the two cases
within the Makefiles, in order to choose appropriate include
directories etc.

Module CFLAGS setting is simplified by using subdir-ccflags-y (available
since 2.6.30) to set them in the top-level Kbuild instead of each
individual module. The disabling of -Wunused-but-set-variable is removed
from the lua and zfs modules. The variable that the Makefile uses is
actually not defined, so this has no effect; and the warning has long
been disabled by the kernel Makefile itself.

The target_cpu definition in module/{zfs,zcommon} is removed as it was
replaced by use of CONFIG_SPARC64 in
  commit 70835c5 ("Unify target_cpu handling")

os/linux/{spl,zfs} are removed from obj-m, as they are not modules in
themselves, but are included by the Makefile in the spl and zfs module
directories. The vestigial Makefiles in os and os/linux are removed.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Arvind Sankar <nivedita@alum.mit.edu>
Closes openzfs#10379
Closes openzfs#10421

configure.ac

@@ -170,13 +170,12 @@ AC_CONFIG_FILES([
 	man/man1/Makefile
 	man/man5/Makefile
 	man/man8/Makefile
+	module/Kbuild
 	module/Makefile
 	module/avl/Makefile
 	module/icp/Makefile
 	module/lua/Makefile
 	module/nvpair/Makefile
-	module/os/Makefile
-	module/os/linux/Makefile
 	module/os/linux/spl/Makefile
 	module/os/linux/zfs/Makefile
 	module/spl/Makefile

copy-builtin

@@ -11,21 +11,6 @@ usage()
 [ "$#" -eq 1 ] || usage
 KERNEL_DIR="$(readlink --canonicalize-existing "$1")"
 
-MODULES=()
-
-# When integrated in to a monolithic kernel the spl module must appear
-# first.  This ensures its module initialization function is run before
-# any of the other module initialization functions which depend on it.
-MODULES+="spl"
-
-for MODULE_DIR in module/* module/os/linux/*
-do
-	[ -d "$MODULE_DIR" ] || continue
-	[ "spl" = "${MODULE_DIR##*/}" ] && continue
-	[ "os" = "${MODULE_DIR#*/}" ] && continue
-	MODULES+=("${MODULE_DIR#*/}")
-done
-
 if ! [ -e 'zfs_config.h' ]
 then
 	echo >&2
@@ -43,12 +28,6 @@ cp --recursive include "$KERNEL_DIR/include/zfs"
 cp --recursive module "$KERNEL_DIR/fs/zfs"
 cp zfs_config.h "$KERNEL_DIR/include/zfs/"
 
-for MODULE in "${MODULES[@]}"
-do
-	sed -i.bak '/obj =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile"
-	sed -i.bak '/src =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile"
-done
-
 cat > "$KERNEL_DIR/fs/zfs/Kconfig" <<"EOF"
 config ZFS
 	tristate "ZFS filesystem support"
@@ -65,27 +44,6 @@ config ZFS
 	  If unsure, say N.
 EOF
 
-{
-	cat <<-"EOF"
-	ZFS_MODULE_CFLAGS  = -I$(srctree)/include/zfs
-	ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/spl
-	ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/zfs
-	ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/kernel
-	ZFS_MODULE_CFLAGS += -include $(srctree)/include/zfs/zfs_config.h
-	ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
-	ZFS_MODULE_CPPFLAGS  = -D_KERNEL
-	ZFS_MODULE_CPPFLAGS += -UDEBUG -DNDEBUG
-	export ZFS_MODULE_CFLAGS ZFS_MODULE_CPPFLAGS
-
-	obj-$(CONFIG_ZFS) :=
-	EOF
-
-	for MODULE in "${MODULES[@]}"
-	do
-		echo 'obj-$(CONFIG_ZFS) += ' "$MODULE/"
-	done
-} > "$KERNEL_DIR/fs/zfs/Kbuild"
-
 add_after()
 {
 	local FILE="$1"

module/.gitignore

@@ -9,6 +9,7 @@
 .*.d
 *.mod
 
+/Kbuild
 /.cache.mk
 /.tmp_versions
 /Module.markers

module/Kbuild.in

@@ -0,0 +1,44 @@
+# When integrated in to a monolithic kernel the spl module must appear
+# first.  This ensures its module initialization function is run before
+# any of the other module initialization functions which depend on it.
+ZFS_MODULES += spl/
+ZFS_MODULES += avl/
+ZFS_MODULES += icp/
+ZFS_MODULES += lua/
+ZFS_MODULES += nvpair/
+ZFS_MODULES += unicode/
+ZFS_MODULES += zcommon/
+ZFS_MODULES += zfs/
+
+# The rest is only relevant when run by kbuild
+ifneq ($(KERNELRELEASE),)
+
+obj-$(CONFIG_ZFS) := $(ZFS_MODULES)
+
+ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
+ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@  @NO_FORMAT_ZERO_LENGTH@
+
+ifneq ($(KBUILD_EXTMOD),)
+zfs_include = @abs_top_srcdir@/include
+ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h
+else
+zfs_include = $(srctree)/include/zfs
+ZFS_MODULE_CFLAGS += -include $(zfs_include)/zfs_config.h
+endif
+
+ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/kernel
+ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/spl
+ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/zfs
+ZFS_MODULE_CFLAGS += -I$(zfs_include)
+ZFS_MODULE_CPPFLAGS += -D_KERNEL
+ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@
+
+ifneq ($(KBUILD_EXTMOD),)
+@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include
+@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
+endif
+
+subdir-asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+subdir-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
+endif

module/Makefile.in

@@ -1,31 +1,7 @@
-obj-m += avl/
-obj-m += icp/
-obj-m += lua/
-obj-m += nvpair/
-obj-m += spl/
-obj-m += os/linux/spl/
-obj-m += unicode/
-obj-m += zcommon/
-obj-m += zfs/
-obj-m += os/linux/zfs/
+include Kbuild
 
 INSTALL_MOD_DIR ?= extra
 
-ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
-ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@  @NO_FORMAT_ZERO_LENGTH@
-ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h
-ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/kernel
-ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/spl
-ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/zfs
-ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include
-ZFS_MODULE_CPPFLAGS += -D_KERNEL
-ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@
-
-@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include
-@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
-
-export ZFS_MODULE_CFLAGS ZFS_MODULE_CPPFLAGS
-
 SUBDIR_TARGETS = icp lua
 
 all: modules
@@ -119,7 +95,7 @@ modules_install: modules_install-@ac_system@
 modules_uninstall-Linux:
 	@# Uninstall the kernel modules
 	kmoddir=$(DESTDIR)$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@
-	list='$(obj-m)'; for objdir in $$list; do \
+	list='$(ZFS_MODULES)'; for objdir in $$list; do \
 		$(RM) -R $$kmoddir/$(INSTALL_MOD_DIR)/$$objdir; \
 	done
 
@@ -129,7 +105,7 @@ modules_uninstall-FreeBSD:
 modules_uninstall: modules_uninstall-@ac_system@
 
 distdir:
-	list='$(obj-m)'; for objdir in $$list; do \
+	list='$(ZFS_MODULES)'; for objdir in $$list os/linux/spl os/linux/zfs; do \
 		(cd @top_srcdir@/module && find $$objdir -name '*.[chS]' | \
 		while read path; do \
 			mkdir -p @abs_top_builddir@/module/$$distdir/$${path%/*}; \

module/avl/Makefile.in

@@ -1,10 +1,10 @@
-src = @abs_top_srcdir@/module/avl
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+endif
 
 MODULE := zavl
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
 $(MODULE)-objs += avl.o

module/icp/Makefile.in

@@ -1,14 +1,17 @@
-src = @abs_top_srcdir@/module/icp
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+icp_include = $(src)/include
+else
+icp_include = $(srctree)/$(src)/include
+endif
 
 MODULE := icp
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-asflags-y := -I@abs_top_srcdir@/module/icp/include
-asflags-y += $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-ccflags-y := -I@abs_top_srcdir@/module/icp/include
-ccflags-y += $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+asflags-y := -I$(icp_include)
+ccflags-y := -I$(icp_include)
 
 $(MODULE)-objs += illumos-crypto.o
 $(MODULE)-objs += api/kcf_cipher.o

module/lua/Makefile.in

@@ -1,16 +1,13 @@
-src = @abs_top_srcdir@/module/lua
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+endif
 
 MODULE := zlua
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-asflags-y += $(ZFS_MODULE_CFLAGS)
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-ccflags-y += -DLUA_USE_LONGLONG
-
-# Suppress unused but set variable warnings often due to ASSERTs
-ccflags-y += $(NO_UNUSED_BUT_SET_VARIABLE)
+ccflags-y := -DLUA_USE_LONGLONG
 
 $(MODULE)-objs += lapi.o
 $(MODULE)-objs += lauxlib.o

module/nvpair/Makefile.in

@@ -1,12 +1,12 @@
-src = @abs_top_srcdir@/module/nvpair
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+endif
 
 MODULE := znvpair
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
 $(MODULE)-objs += nvpair.o
 $(MODULE)-objs += fnvpair.o
 $(MODULE)-objs += nvpair_alloc_spl.o

module/os/linux/zfs/Makefile.in

@@ -5,8 +5,6 @@
 # Suppress unused-value warnings in sparc64 architecture headers
 ccflags-$(CONFIG_SPARC64) += -Wno-unused-value
 
-ccflags-y += -I@abs_top_srcdir@/module/os/linux/zfs
-
 $(MODULE)-objs += ../os/linux/zfs/abd_os.o
 $(MODULE)-objs += ../os/linux/zfs/arc_os.o
 $(MODULE)-objs += ../os/linux/zfs/mmp_os.o

module/spl/Makefile.in

@@ -1,11 +1,13 @@
-src = @abs_top_srcdir@/module/spl
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+mfdir = $(obj)
+else
+mfdir = $(srctree)/$(src)
+endif
 
 MODULE := spl
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
-
--include @abs_top_builddir@/module/os/linux/spl/Makefile
+include $(mfdir)/../os/linux/spl/Makefile

module/unicode/Makefile.in

@@ -1,11 +1,11 @@
-src = @abs_top_srcdir@/module/unicode
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
+endif
 
 MODULE := zunicode
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
 $(MODULE)-objs += u8_textprep.o
 $(MODULE)-objs += uconv.o

module/zcommon/Makefile.in

@@ -1,13 +1,12 @@
-src = @abs_top_srcdir@/module/zcommon
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
-target_cpu = @target_cpu@
+endif
 
 MODULE := zcommon
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
 # Suppress unused-value warnings in sparc64 architecture headers
 ccflags-$(CONFIG_SPARC64) += -Wno-unused-value
 

module/zfs/Makefile.in

@@ -1,16 +1,15 @@
-src = @abs_top_srcdir@/module/zfs
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
 obj = @abs_builddir@
-target_cpu = @target_cpu@
+mfdir = $(obj)
+else
+mfdir = $(srctree)/$(src)
+endif
 
 MODULE := zfs
 
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
-
-# Suppress unused but set variable warnings often due to ASSERTs
-ccflags-y += $(NO_UNUSED_BUT_SET_VARIABLE)
-
 # Suppress unused-value warnings in sparc64 architecture headers
 ccflags-$(CONFIG_SPARC64) += -Wno-unused-value
 
@@ -150,4 +149,4 @@ ifeq ($(CONFIG_ALTIVEC),y)
 $(obj)/vdev_raidz_math_powerpc_altivec.o: c_flags += -maltivec
 endif
 
--include @abs_top_builddir@/module/os/linux/zfs/Makefile
+include $(mfdir)/../os/linux/zfs/Makefile