Refactor build system, remove hvt compile-time specialization

This is a large change, and the majority of it is effectively a full
re-write of the build system. The actual removal of hvt compile-time
specialization is fairly straightforward.

User-visible changes:

- 'configure.sh' now needs to be run manually before running 'make',
  this is consistent with POLA.
- conversely, 'make clean' no longer cleans Makeconf. Use 'distclean' or
  'clobber' for that.
- 'configure.sh' will now print the targets that can (will) be built on
  this system. The strategy is still "build everything we can", however
  I have disabled Genode on all systems except Linux due to toolchain
  issues.
- You can now build a subset of targets from the top-level 'make', by
  specifying 'CONFIG_XXX=' (disable) or 'CONFIG_XXX=1' (enable) either
  on the command line, or editing the generated Makeconf.
- Makefiles use silent rules by default. To get the old verbose ones
  back, use 'make V=1'.
- The 'solo5-hvt' tender is no longer "specialized" to the unikernel.
  We build two tenders, 'solo5-hvt' with all non-debug modules
  configured and 'solo5-hvt-debug' with additional debug modules (gdb,
  dumpcore where available).
- 'solo5-hvt-configure' is kept around for now for backward
  compatibility with OPAM/MirageOS but is essentially a NOP.

Developer-visible changes:

- The build system now has proper support for auto-generation of
  dependencies. This means you can safely edit source files, run make
  and be sure you will get a complete incremental build.
- Makefiles have been refactored to use common best practices, remove
  repetition, consistent variable names and clear interfaces between
  configure.sh/Makeconf/Makefiles, all the while keeping them simple
  enough to understand for me on a Monday morning before coffee. I.e.
  limit use of macros, eval, etc.
- hvt tender modules are no longer defined by compile-time flags,
  instead a dynamic array is placed into a special ELF section
  (.modules).  This means that a hvt tender binary can be combined from
  an arbitrary set of hvt_module_XXX object files, which is the right
  way to do things going forward and also simplifies the build system
  (not needing to build multiple targets from the same set of sources).

Shortcomings / TODOs:

- Dependency files (*.d) are stored in-tree. I spent several days on
  trying to figure out how to get them to work out of tree, but in
  combination with the non-recursive use of subdirectories in 'bindings'
  I could not figure out the required Makefile magic.
- HVT_DROP_PRIVILEGES=0 is non-functional with the new modules
  arrangement, but needs a re-design anyway.

Other changes included as part of this PR:

- Revert privilege dropping on FreeBSD (see discussion in Solo5#282).
- The build system changes effectively implement option 1 in Solo5#292, i.e.
  on x86_64 -m no-red-zone is only used for bindings, not for
  application code.
- tests/tests.bats has been refactored for DRY as it was getting totally
  unmaintainable.

.gitignore

@@ -1,19 +1,15 @@
 *.o
+*.d
 *.lib.so
-*.swp
-*.pcap
+*.pc
 Makeconf
-cscope.*
 tags
 include/crt
 tests/*/*.hvt
 tests/*/*.spt
 tests/*/*.virtio
 tests/*/*.muen
 tests/*/*.genode
-tests/*/solo5-hvt
-tests/*/Makefile.solo5-hvt
-tenders/hvt/Makefile.solo5-hvt
-tenders/hvt/_build-solo5-hvt/
 tenders/hvt/solo5-hvt
+tenders/hvt/solo5-hvt-debug
 tenders/spt/solo5-spt

.travis.yml

@@ -5,4 +5,4 @@ addons:
         packages:
             - qemu-system-x86
             - libseccomp-dev
-script: make && tests/run-tests.sh
+script: ./configure.sh && make && tests/run-tests.sh

GNUmakefile

@@ -16,166 +16,108 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-export TOP=$(abspath .)
-$(TOP)/Makeconf:
-	./configure.sh
+export TOPDIR := $(abspath .)
+$(TOPDIR)/Makeconf:
+	$(error Makeconf not found, please run ./configure.sh)
 include Makefile.common
 
-.PHONY: all
-all: hvt spt virtio muen genode
-.DEFAULT_GOAL := all
-.NOTPARALLEL: hvt virtio muen genode
+SUBDIRS := bindings tenders/hvt tenders/spt tests
 
-.PHONY: virtio
-virtio:
-ifeq ($(BUILD_VIRTIO), yes)
-	$(MAKE) -C bindings virtio
-	$(MAKE) -C tests virtio
-endif
+tests: bindings
 
-.PHONY: hvt
-hvt:
-ifeq ($(BUILD_HVT), yes)
-	$(MAKE) -C bindings hvt
-	$(MAKE) -C tenders/hvt
-	$(MAKE) -C tests hvt
-endif
+.PHONY: $(SUBDIRS)
 
-.PHONY: spt
-spt:
-ifeq ($(BUILD_SPT), yes)
-	$(MAKE) -C bindings spt
-	$(MAKE) -C tenders/spt
-	$(MAKE) -C tests spt
-endif
-
-.PHONY: muen
-muen:
-ifeq ($(BUILD_MUEN), yes)
-	$(MAKE) -C bindings muen
-	$(MAKE) -C tests muen
-endif
-
-.PHONY: genode
-genode:
-ifeq ($(BUILD_GENODE), yes)
-	$(MAKE) -C bindings genode
-	$(MAKE) -C tests genode
-endif
+.PHONY: all
+all: $(SUBDIRS)
+.DEFAULT_GOAL := all
 
-.PHONY: clean
-clean:
-	$(MAKE) -C bindings clean
-ifeq ($(BUILD_HVT), yes)
-	$(MAKE) -C tenders/hvt clean
-endif
-ifeq ($(BUILD_SPT), yes)
-	$(MAKE) -C tenders/spt clean
-endif
-	$(MAKE) -C tests clean
+$(SUBDIRS):
+	@echo "MAKE $@"
+	$(MAKE) -C $@ $(MAKECMDGOALS) $(SUBOVERRIDE)
+
+.PHONY: clean before-clean
+# Ensure that a top-level "make clean" always cleans *all* possible build
+# products and not some subset dependent on the setting of $(BUILD_*).
+before-clean:
+	$(eval export SUBOVERRIDE := CONFIG_HVT=1 CONFIG_SPT=1 CONFIG_VIRTIO=1 CONFIG_MUEN=1 CONFIG_GENODE=1)
+clean: before-clean $(SUBDIRS)
+	@echo "CLEAN solo5"
 	$(RM) solo5-bindings-virtio.pc
 	$(RM) solo5-bindings-hvt.pc
 	$(RM) solo5-bindings-muen.pc
 	$(RM) solo5-bindings-genode.pc
-	$(RM) -r include/crt
-	$(RM) Makeconf
-
-PREFIX?=/nonexistent # Fail if not run from OPAM
-OPAM_BINDIR=$(PREFIX)/bin
-OPAM_HVT_LIBDIR=$(PREFIX)/lib/solo5-bindings-hvt
-OPAM_HVT_INCDIR=$(PREFIX)/include/solo5-bindings-hvt
-OPAM_VIRTIO_LIBDIR=$(PREFIX)/lib/solo5-bindings-virtio
-OPAM_VIRTIO_INCDIR=$(PREFIX)/include/solo5-bindings-virtio
-OPAM_MUEN_LIBDIR=$(PREFIX)/lib/solo5-bindings-muen
-OPAM_MUEN_INCDIR=$(PREFIX)/include/solo5-bindings-muen
-OPAM_GENODE_LIBDIR=$(PREFIX)/lib/solo5-bindings-genode
-OPAM_GENODE_INCDIR=$(PREFIX)/include/solo5-bindings-genode
-OPAM_SPT_LIBDIR=$(PREFIX)/lib/solo5-bindings-spt
-OPAM_SPT_INCDIR=$(PREFIX)/include/solo5-bindings-spt
-
-# We want the MD CFLAGS, LDFLAGS and LD in the .pc file, where they can be
-# picked up by the Mirage tool / other downstream consumers.
-%.pc: %.pc.in
-	sed <$< > $@ \
-	    -e 's#!CFLAGS!#$(MD_CFLAGS)#g;' \
-	    -e 's#!LDFLAGS!#$(LDFLAGS)#g;' \
-	    -e 's#!GENODE_APP_LDFLAGS!#$(GENODE_APP_LDFLAGS)#g;' \
-	    -e 's#!LD!#$(LD)#g;' \
 
-.PHONY: opam-virtio-install
-opam-virtio-install: solo5-bindings-virtio.pc virtio
-	mkdir -p $(OPAM_VIRTIO_INCDIR) $(OPAM_VIRTIO_LIBDIR)
-	cp -R include/. $(OPAM_VIRTIO_INCDIR)
-	cp bindings/virtio/solo5_virtio.o bindings/virtio/solo5_virtio.lds $(OPAM_VIRTIO_LIBDIR)
-	mkdir -p $(OPAM_BINDIR)
-	cp scripts/virtio-mkimage/solo5-virtio-mkimage.sh ${OPAM_BINDIR}/solo5-virtio-mkimage
-	cp scripts/virtio-run/solo5-virtio-run.sh ${OPAM_BINDIR}/solo5-virtio-run
-	mkdir -p $(PREFIX)/lib/pkgconfig
-	cp solo5-bindings-virtio.pc $(PREFIX)/lib/pkgconfig
-
-.PHONY: opam-virtio-uninstall
-opam-virtio-uninstall:
-	$(RM) -r $(OPAM_VIRTIO_INCDIR) $(OPAM_VIRTIO_LIBDIR)
-	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-virtio.pc
-	$(RM) ${OPAM_BINDIR}/solo5-mkimage
-	$(RM) ${OPAM_BINDIR}/solo5-run-virtio
-
-.PHONY: opam-hvt-install
-opam-hvt-install: solo5-bindings-hvt.pc hvt
-	mkdir -p $(OPAM_HVT_INCDIR) $(OPAM_HVT_LIBDIR)
-	cp -R include/. $(OPAM_HVT_INCDIR)
-	cp bindings/hvt/solo5_hvt.o bindings/hvt/solo5_hvt.lds $(OPAM_HVT_LIBDIR)
-	mkdir -p $(OPAM_HVT_LIBDIR)/src
-	cp -R tenders/hvt/*.[ch] include/solo5/hvt_abi.h $(OPAM_HVT_LIBDIR)/src
-	mkdir -p $(OPAM_BINDIR)
-	cp tenders/hvt/solo5-hvt-configure $(OPAM_BINDIR)
-	mkdir -p $(PREFIX)/lib/pkgconfig
-	cp solo5-bindings-hvt.pc $(PREFIX)/lib/pkgconfig
+.PHONY: distclean
+distclean: MAKECMDGOALS := clean
+distclean: clean
+	@echo DISTCLEAN solo5
+	-[ -d include/crt ] && $(RM) -r include/crt
+	$(RM) Makeconf
 
-.PHONY: opam-hvt-uninstall
-opam-hvt-uninstall:
-	$(RM) -r $(OPAM_HVT_INCDIR) $(OPAM_HVT_LIBDIR)
-	$(RM) $(OPAM_BINDIR)/solo5-hvt-configure
-	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-hvt.pc
+.PHONY: force-install
+install-opam-%: MAKECMDGOALS :=
+install-opam-%: all solo5-bindings-%.pc force-install
+	@echo INSTALL solo5
+	@[ -d "$(PREFIX)" -a -d "$(PREFIX)/bin" ] || \
+	    (echo "error: PREFIX not set or incorrect"; false)
+	mkdir -p $(PREFIX)/lib/pkgconfig \
+	    $(PREFIX)/lib/solo5-bindings-$* \
+	    $(PREFIX)/include/solo5-bindings-$*/solo5 \
+	    $(PREFIX)/include/solo5-bindings-$*/crt
+	cp -R include/solo5/ include/crt/ $(PREFIX)/include/solo5-bindings-$*
+	cp bindings/$*/solo5_$*.o bindings/$*/solo5_$*.lds \
+	    $(PREFIX)/lib/solo5-bindings-$*
+	cp solo5-bindings-$*.pc $(PREFIX)/lib/pkgconfig
+ifdef CONFIG_HVT
+	cp tenders/hvt/solo5-hvt tenders/hvt/solo5-hvt-configure $(PREFIX)/bin
+	[ -f tenders/hvt/solo5-hvt-debug ] && \
+	    cp tenders/hvt/solo5-hvt-debug $(PREFIX)/bin
+endif
+ifdef CONFIG_SPT
+	cp tenders/spt/solo5-spt $(PREFIX)/bin
+endif
+ifdef CONFIG_VIRTIO
+	cp scripts/virtio-mkimage/solo5-virtio-mkimage.sh \
+	    $(PREFIX)/bin/solo5-virtio-mkimage
+	cp scripts/virtio-run/solo5-virtio-run.sh \
+	    $(PREFIX)/bin/solo5-virtio-run
+endif
 
-.PHONY: opam-muen-install
-opam-muen-install: solo5-bindings-muen.pc muen
-	mkdir -p $(OPAM_MUEN_INCDIR) $(OPAM_MUEN_LIBDIR)
-	cp -R include/. $(OPAM_MUEN_INCDIR)
-	cp bindings/muen/solo5_muen.o bindings/muen/solo5_muen.lds $(OPAM_MUEN_LIBDIR)
-	mkdir -p $(PREFIX)/lib/pkgconfig
-	cp solo5-bindings-muen.pc $(PREFIX)/lib/pkgconfig
+.PHONY: force-uninstall
+uninstall-opam-%: force-uninstall
+	@echo UNINSTALL solo5
+	@[ -d "$(PREFIX)" -a -d "$(PREFIX)/bin" ] || \
+	    (echo "error: PREFIX not set or incorrect"; false)
+	-[ -d "$(PREFIX)/include/solo5-bindings-$*/solo5" ] && \
+	    $(RM) -r $(PREFIX)/include/solo5-bindings-$*/solo5
+	-[ -d "$(PREFIX)/include/solo5-bindings-$*/crt" ] && \
+	    $(RM) -r $(PREFIX)/include/solo5-bindings-$*/crt
+	$(RM) $(PREFIX)/lib/solo5-bindings-$*/solo5_$*.o \
+	    $(PREFIX)/lib/solo5-bindings-$*/solo5_$*.lds
+	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-$*.pc
+ifdef CONFIG_HVT
+	$(RM) $(PREFIX)/bin/solo5-hvt $(PREFIX)/bin/solo5-hvt-debug \
+	    $(PREFIX)/bin/solo5-hvt-configure
+endif
+ifdef CONFIG_SPT
+	$(RM) $(PREFIX)/bin/solo5-spt
+endif
+ifdef CONFIG_VIRTIO
+	$(RM) $(PREFIX)/bin/solo5-virtio-mkimage
+	$(RM) $(PREFIX)/bin/solo5-virtio-run
+endif
 
-.PHONY: opam-muen-uninstall
-opam-muen-uninstall:
-	$(RM) -r $(OPAM_MUEN_INCDIR) $(OPAM_MUEN_LIBDIR)
-	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-muen.pc
+# The following targets are kept for backwards compatibility, as otherwise
+# upgrading existing OPAM switches will fail. They should be removed at some
+# point, along with the dummy solo5-hvt-configure.
+opam-hvt-uninstall: uninstall-opam-hvt ;
+
+opam-spt-uninstall: uninstall-opam-spt ;
 
-.PHONY: opam-genode-install
-opam-genode-install: solo5-bindings-genode.pc genode
-	mkdir -p $(OPAM_GENODE_INCDIR) $(OPAM_GENODE_LIBDIR)
-	cp -R include/. $(OPAM_GENODE_INCDIR)
-	cp bindings/genode/solo5.lib.so bindings/genode/genode_dyn.ld $(OPAM_GENODE_LIBDIR)
-	mkdir -p $(PREFIX)/lib/pkgconfig
-	cp solo5-bindings-genode.pc $(PREFIX)/lib/pkgconfig
+opam-virtio-uninstall: uninstall-opam-virtio ;
 
-.PHONY: opam-genode-uninstall
-opam-genode-uninstall:
-	$(RM) -r $(OPAM_GENODE_INCDIR) $(OPAM_GENODE_LIBDIR)
-	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-genode.pc
+opam-muen-uninstall: uninstall-opam-muen ;
 
-.PHONY: opam-spt-install
-opam-spt-install: solo5-bindings-spt.pc spt
-	mkdir -p $(OPAM_SPT_INCDIR) $(OPAM_SPT_LIBDIR)
-	cp -R include/. $(OPAM_SPT_INCDIR)
-	cp bindings/spt/solo5_spt.o bindings/spt/solo5_spt.lds $(OPAM_SPT_LIBDIR)
-	mkdir -p $(PREFIX)/lib/pkgconfig
-	cp solo5-bindings-spt.pc $(PREFIX)/lib/pkgconfig
-	mkdir -p $(OPAM_BINDIR)
-	cp tenders/spt/solo5-spt ${OPAM_BINDIR}/solo5-spt
+opam-genode-uninstall: uninstall-opam-genode ;
 
-.PHONY: opam-spt-uninstall
-opam-spt-uninstall:
-	$(RM) -r $(OPAM_SPT_INCDIR) $(OPAM_SPT_LIBDIR)
-	$(RM) $(PREFIX)/lib/pkgconfig/solo5-bindings-spt.pc
-	$(RM) ${OPAM_BINDIR}/solo5-spt
+$(V).SILENT:

Makefile.common

@@ -16,31 +16,87 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-# This Makefile defines global defaults for building Solo5 and the in-tree test
-# programs. These can be overriden either on the command line, or via Makeconf
-# generated by configure.sh.
-include $(TOP)/Makeconf
-
-# Be clear about some common programs we use!
-AS?=as
-CC?=cc
-LD?=ld
-OBJCOPY?=objcopy
-
-# Exported to OPAM via pkg-config.
-MD_CFLAGS=-ffreestanding -fstack-protector-strong ${HOST_CFLAGS}
-ifeq ($(TARGET_ARCH), x86_64)
-MD_CFLAGS+=-mno-red-zone
+# This Makefile defines common rules for building Solo5 and the in-tree test
+# programs. Variables starting with CONFIG_ and MAKECONF_ are sourced from the
+# Makeconf generated by configure.sh, and MAY be overriden by the user on the
+# top-level "make" command line.
+ifndef TOPDIR
+$(error TOPDIR must be set, run $(MAKE) from the top of the source tree or set it manually)
 endif
+include $(TOPDIR)/Makeconf
 
-# Likewise.
-COMMON_LDFLAGS=-nostdlib -z max-page-size=0x1000
-LDFLAGS=$(COMMON_LDFLAGS) -static $(HOST_LDFLAGS)
-# CFLAGS used for building bindings/ and in-tree tests.
-INCDIR=-isystem $(TOP)/include/crt -I$(TOP)/include/solo5
-CFLAGS=$(MD_CFLAGS) -std=gnu99 -Wall -Wextra -Werror -O2 -g $(INCDIR)
-
-# Genode linking is a special case
-GENODE_COMMON_LDFLAGS = $(COMMON_LDFLAGS) -shared -gc-sections --eh-frame-hdr
-GENODE_LIB_LDFLAGS=$(GENODE_COMMON_LDFLAGS) --entry=0x0 -T $(TOP)/bindings/genode/genode_rel.ld
-GENODE_APP_LDFLAGS=$(GENODE_COMMON_LDFLAGS) -Ttext=0x01000000 --dynamic-linker=ld.lib.so -rpath-link=.
+#
+# The following variables and recipes apply to building bindings and
+# tests (applications).
+#
+CC := $(MAKECONF_CC)
+CFLAGS := -std=gnu99 -Wall -Wextra -Werror -O2 -g
+CFLAGS += -ffreestanding -fstack-protector-strong $(MAKECONF_CFLAGS)
+CPPFLAGS := -isystem $(TOPDIR)/include/crt -I$(TOPDIR)/include/solo5
+LD := $(MAKECONF_LD)
+LDFLAGS := -nostdlib -z max-page-size=0x1000 -static $(MAKECONF_LDFLAGS)
+OBJCOPY := objcopy
+
+# Genode application compiling and linking is a special case
+GENODE_APP_CFLAGS := $(CFLAGS) -fPIC
+GENODE_APP_LDFLAGS := -nostdlib -z max-page-size=0x1000 -shared -gc-sections \
+    --eh-frame-hdr -Ttext=0x01000000 --dynamic-linker=ld.lib.so -rpath-link=.
+
+# Also used by the HOST_ rules below.
+DEPFLAGS = -MT $@ -MMD -MP -MF $*.Td
+
+define COMPILE.c
+	@echo "CC $<"
+	$(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
+	mv -f $*.Td $*.d && touch $@
+endef
+
+define COMPILE.S
+	@echo "AS $<"
+	$(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -DASM_FILE -c $< -o $@
+	mv -f $*.Td $*.d && touch $@
+endef
+
+#
+# The following variables and recpies apply to building tenders, i.e.
+# artifacts built to run on the (Solo5 tender's) *host*.
+#
+HOSTCC := $(MAKECONF_CC)
+HOSTCFLAGS := -Wall -Werror -std=c99 -O2 -g
+HOSTCPPFLAGS := -I$(TOPDIR)/include/solo5
+HOSTLDFLAGS :=
+HOSTLDLIBS :=
+
+define HOSTCOMPILE.c
+	@echo "HOSTCC $<"
+	$(HOSTCC) $(DEPFLAGS) $(HOSTCFLAGS) $(HOSTCPPFLAGS) -c $< -o $@
+	mv -f $*.Td $*.d && touch $@
+endef
+
+define HOSTCOMPILE.S
+	@echo "HOSTAS $<"
+	$(HOSTCC) $(DEPFLAGS) $(HOSTCFLAGS) $(HOSTCPPFLAGS) -DASM_FILE -c $< -o $@
+	mv -f $*.Td $*.d && touch $@
+endef
+
+define HOSTLINK
+	@echo "HOSTLINK $@"
+	$(HOSTCC) $(HOSTLDFLAGS) $^ $(HOSTLDLIBS) -o $@
+endef
+
+PC_CFLAGS := -ffreestanding -fstack-protector-strong $(MAKECONF_CFLAGS)
+PC_GENODE_CFLAGS := $(GENODE_APP_CFLAGS)
+PC_LD := $(MAKECONF_LD)
+PC_LDFLAGS := $(LDFLAGS)
+PC_GENODE_LDFLAGS := $(GENODE_APP_LDFLAGS)
+
+%.pc: %.pc.in
+	@echo SUBST $@
+	sed <$< > $@ \
+	    -e 's#!PC_CFLAGS!#$(PC_CFLAGS)#g;' \
+	    -e 's#!PC_GENODE_CFLAGS!#$(PC_GENODE_CFLAGS)#g;' \
+	    -e 's#!PC_LD!#$(PC_LD)#g;' \
+	    -e 's#!PC_LDFLAGS!#$(PC_LDFLAGS)#g;' \
+	    -e 's#!PC_GENODE_LDFLAGS!#$(PC_GENODE_LDFLAGS)#g;' \
+
+.PRECIOUS: %.pc