Merge pull request snabbco#70 from lukego/split-build

Add VM generated code to repo, support simplified build

.travis.yml

@@ -1,9 +1,10 @@
 language: nix
 sudo: false
 env:
-  - test=test-O3
-  - test=test-O2
-  - test=test-O1
-  - test=test-nojit
+  - nix-build -A test-O3
+  - nix-build -A test-O2
+  - nix-build -A test-O1
+  - nix-build -A test-nojit
+  - nix-build -A check-generated-code --arg check true
 script:
-  - nix-build -A $test
+  - $test

Makefile

@@ -1,5 +1,5 @@
 ##############################################################################
-# LuaJIT top level Makefile for installation. Requires GNU Make.
+# RaptorJIT top level Makefile for installation. Requires GNU Make.
 #
 # Please read doc/install.html before changing any variables!
 #
@@ -13,10 +13,10 @@
 # Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
 ##############################################################################
 
-MAJVER=  2
-MINVER=  1
+MAJVER=  1
+MINVER=  0
 RELVER=  0
-PREREL=  -beta2
+PREREL=  -alpha1
 VERSION= $(MAJVER).$(MINVER).$(RELVER)$(PREREL)
 ABIVER=  5.1
 
@@ -109,12 +109,12 @@ endif
 INSTALL_DEP= src/luajit
 
 default all $(INSTALL_DEP):
-	@echo "==== Building LuaJIT $(VERSION) ===="
+	@echo "==== Building RaptorJIT $(VERSION) ===="
 	$(MAKE) -C src
-	@echo "==== Successfully built LuaJIT $(VERSION) ===="
+	@echo "==== Successfully built RaptorJIT $(VERSION) ===="
 
 install: $(INSTALL_DEP)
-	@echo "==== Installing LuaJIT $(VERSION) to $(PREFIX) ===="
+	@echo "==== Installing RaptorJIT $(VERSION) to $(PREFIX) ===="
 	$(MKDIR) $(INSTALL_DIRS)
 	cd src && $(INSTALL_X) $(FILE_T) $(INSTALL_T)
 	cd src && test -f $(FILE_A) && $(INSTALL_F) $(FILE_A) $(INSTALL_STATIC) || :
@@ -130,7 +130,7 @@ install: $(INSTALL_DEP)
 	  $(RM) $(FILE_PC).tmp
 	cd src && $(INSTALL_F) $(FILES_INC) $(INSTALL_INC)
 	cd src/jit && $(INSTALL_F) $(FILES_JITLIB) $(INSTALL_JITLIB)
-	@echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ===="
+	@echo "==== Successfully installed RaptorJIT $(VERSION) to $(PREFIX) ===="
 	@echo ""
 	@echo "Note: the development releases deliberately do NOT install a symlink for luajit"
 	@echo "You can do this now by running this command (with sudo):"
@@ -140,7 +140,7 @@ install: $(INSTALL_DEP)
 
 
 uninstall:
-	@echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ===="
+	@echo "==== Uninstalling RaptorJIT $(VERSION) from $(PREFIX) ===="
 	$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC)
 	for file in $(FILES_JITLIB); do \
 	  $(UNINSTALL) $(INSTALL_JITLIB)/$$file; \
@@ -150,13 +150,16 @@ uninstall:
 	  done
 	$(LDCONFIG) $(INSTALL_LIB)
 	$(RMDIR) $(UNINSTALL_DIRS) || :
-	@echo "==== Successfully uninstalled LuaJIT $(VERSION) from $(PREFIX) ===="
+	@echo "==== Successfully uninstalled RaptorJIT $(VERSION) from $(PREFIX) ===="
 
 ##############################################################################
 
 clean:
 	$(MAKE) -C src clean
 
+bootstrapclean:
+	$(MAKE) -C src bootstrapclean
+
 .PHONY: all install clean
 
 ##############################################################################

README.md

@@ -68,16 +68,31 @@ details in discussions then these are the places to look for answers.
 
 The [AnandTech review of the Haswell microarchitecture](http://www.anandtech.com/show/6355/intels-haswell-architecture) is also excellent lighter reading.
 
-### Compilation
+### Compilation for users
 
-RaptorJIT uses [nix](http://nixos.org/nix/) to define a reproducible
-build environment that includes Clang for C and LuaJIT 2.0 for
-bootstrapping (see [default.nix](default.nix)). The recommended way to
-build RaptorJIT is with nix, which provides the dependencies
-automatically, but you can build manually if you prefer.
+Simple build:
 
-Building with nix will be slow the first time due to downloading
-toolchains and related dependencies. This is all cached for future
+```shell
+$ make  # requires LuaJIT (2.0 or 2.1) to run DynASM
+```
+
+Alternative if you don't have LuaJIT available and you are building a
+pristine copy from the master branch:
+
+```shell
+$ make reusevm  # Reuse reference copy of the generated VM code
+$ make          # Does not require LuaJIT now
+```
+
+### Compilation for VM hackers
+
+RaptorJIT uses [Nix](http://nixos.org/nix/) to provide a reference
+build environment. You can use Nix to build/test/benchmark RaptorJIT
+with suitable versions of all dependencies provided.
+
+Note: Building with nix will be slow the first time because it
+downloads the exact reference versions of the toolchain (clang, etc)
+and all dependencies (glibc, etc). This is all cached for future
 builds.
 
 #### Build with nix

check-generated-code.nix

@@ -0,0 +1,31 @@
+# Check that generated sources match the repo version.
+{ pkgs, raptorjit }:
+with pkgs; with lib;
+
+# Generated files that are kept in tree.
+let generatedFiles =
+  "lj_bcdef.h lj_ffdef.h lj_libdef.h lj_recdef.h lj_folddef.h host/buildvm_arch.h";
+in
+
+overrideDerivation raptorjit (as:
+  {
+    preBuild = ''
+      pushd src
+      mkdir old
+      for f in ${generatedFiles}; do
+        cp $f old/
+      done
+      popd
+    '' + as.preBuild;
+    checkPhase = ''
+      pushd src
+      mkdir new
+      for f in ${generatedFiles}; do
+        cp $f new/
+      done
+      echo "Checking that in-tree generated VM code is up-to-date..."
+      diff -u old new || (echo "Error: Stale generated code"; exit 1)
+      popd
+    '';
+    doCheck = true;
+  })

default.nix

@@ -9,12 +9,14 @@
 
 { pkgs ? (import ./pkgs.nix) {}
 , source ? pkgs.lib.cleanSource ./.
-, version ? "dev" }:
+, version ? "dev"
+, check ? false }:
 
 let
-  callPackage = (pkgs.lib.callPackageWith { inherit pkgs; inherit source; inherit version; });
+  callPackage = (pkgs.lib.callPackageWith { inherit pkgs source version; });
   raptorjit = (callPackage ./raptorjit.nix {});
-  test = name: args: (callPackage ./test.nix { inherit raptorjit; inherit name; inherit args; });
+  test = name: args: (callPackage ./test.nix { inherit raptorjit name args; });
+  check-generated-code = (callPackage ./check-generated-code.nix { inherit raptorjit; });
 in
 
 # Build RaptorJIT and run mulitple test suites.
@@ -24,5 +26,6 @@ in
   test-O2    = test "O2"    "-O2";
   test-O1    = test "O1"    "-O1";
   test-nojit = test "nojit" "-joff";
-}
+} //
+(if check then { inherit check-generated-code; } else {})
 

src/.gitignore

@@ -1,7 +1 @@
 luajit
-lj_bcdef.h
-lj_ffdef.h
-lj_libdef.h
-lj_recdef.h
-lj_folddef.h
-lj_vm.[sS]

src/Makefile

@@ -10,8 +10,8 @@
 # Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
 ##############################################################################
 
-MAJVER=  2
-MINVER=  1
+MAJVER=  1
+MINVER=  0
 RELVER=  0
 ABIVER=  5.1
 NODOTABIVER= 51
@@ -24,14 +24,6 @@ NODOTABIVER= 51
 # removing the '#' in front of them. Make sure you force a full recompile
 # with "make clean", followed by "make" if you change any options.
 #
-DEFAULT_CC = clang
-#
-# LuaJIT builds as a native 32 or 64 bit binary by default.
-CC= $(DEFAULT_CC)
-#
-# Use this if you want to force a 32 bit build on a 64 bit multilib OS.
-#CC= $(DEFAULT_CC) -m32
-#
 # Since the assembler part does NOT maintain a frame pointer, it's pointless
 # to slow down the C part by not omitting it. Debugging, tracebacks and
 # unwinding are not affected -- the assembler part has frame unwind
@@ -161,8 +153,8 @@ TARGET_LD= $(CROSS)$(CC)
 TARGET_AR= $(CROSS)ar rcus 2>/dev/null
 
 TARGET_LIBPATH= $(or $(PREFIX),/usr/local)/$(or $(MULTILIB),lib)
-TARGET_SONAME= libluajit-$(ABIVER).so.$(MAJVER)
-TARGET_DYLIBNAME= libluajit-$(ABIVER).$(MAJVER).dylib
+TARGET_SONAME= libraptorjit-$(ABIVER).so.$(MAJVER)
+TARGET_DYLIBNAME= libraptorjit-$(ABIVER).$(MAJVER).dylib
 TARGET_DYLIBPATH= $(TARGET_LIBPATH)/$(TARGET_DYLIBNAME)
 TARGET_DLLNAME= lua$(NODOTABIVER).dll
 TARGET_XSHLDFLAGS= -shared -fPIC -Wl,-soname,$(TARGET_SONAME)
@@ -250,28 +242,25 @@ LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \
 LJVMCORE_O= $(LJVM_O) $(LJCORE_O)
 LJVMCORE_DYNO= $(LJVMCORE_O:.o=_dyn.o)
 
-LIB_VMDEF= jit/vmdef.lua
-LIB_VMDEFP= $(LIB_VMDEF)
-
-LUAJIT_O= luajit.o
-LUAJIT_A= libluajit.a
-LUAJIT_SO= libluajit.so
-LUAJIT_T= luajit
+LUAJIT_O= raptorjit.o
+LUAJIT_A= raptorjit.a
+LUAJIT_SO= libraptorjit.so
+LUAJIT_T= raptorjit
 
 ALL_T= $(LUAJIT_T) $(LUAJIT_A) $(LUAJIT_SO) $(HOST_T)
 ALL_HDRGEN= lj_bcdef.h lj_ffdef.h lj_libdef.h lj_recdef.h lj_folddef.h \
 	    host/buildvm_arch.h
-ALL_GEN= $(LJVM_S) $(ALL_HDRGEN) $(LIB_VMDEFP)
+ALL_GEN= $(LJVM_S) $(ALL_HDRGEN)
 WIN_RM= *.obj *.lib *.exp *.dll *.exe *.manifest *.pdb *.ilk
-ALL_RM= $(ALL_T) $(ALL_GEN) *.o host/*.o $(WIN_RM)
+ALL_RM= $(ALL_T) *.o host/*.o $(WIN_RM)
 
 ##############################################################################
 # Build mode handling.
 ##############################################################################
 
 TARGET_O= $(LUAJIT_A)
 TARGET_T= $(LUAJIT_T) $(LUAJIT_SO)
-TARGET_DEP= $(LIB_VMDEF) $(LUAJIT_SO)
+TARGET_DEP= $(LUAJIT_SO)
 
 Q= @
 E= @echo
@@ -286,6 +275,12 @@ default all:	$(TARGET_T)
 
 clean:
 	$(HOST_RM) $(ALL_RM)
+	$(HOST_RM) $(ALL_GEN)
+
+reusevm:
+	cd reusevm; \
+	cp -r * ../
+	@echo "Copied reference VM. Ready to build."
 
 libbc:
 	./$(LUAJIT_T) host/genlibbc.lua -o host/buildvm_libbc.h $(LJLIB_C)
@@ -305,13 +300,28 @@ depend:
 	  test -s $$file || $(HOST_RM) $$file; \
 	  done
 
-.PHONY: default all clean libbc depend
+.PHONY: default all clean libbc depend reusevm
 
 ##############################################################################
 # Rules for generated files.
 ##############################################################################
 
 host/buildvm_arch.h: $(DASM_DASC) $(DASM_DEP) $(DASM_DIR)/*.lua
+	@command -v $(DASM) 2>/dev/null || { \
+	  echo "Error: Missing dependency (luajit) for bootstrapping the VM."; \
+	  echo "";                                                             \
+	  echo "Here are your options to build RaptorJIT:";                    \
+	  echo "  make reusevm     # copy reference VM from reusevm/";          \
+	  echo "  (install luajit) # satisfy the dependency";                   \
+	  echo "  nix-build        # use nix to satisfy the dependencies";      \
+	  echo "";                                                             \
+	  echo "Warning:";                                                     \
+	  echo "  Only reusevm when running a pristine copy of RaptorJIT from";\
+	  echo "  a repository that keeps the reference VM up-to-date.";       \
+	  echo "  (Otherwise you might mix some stale code into your build.)"; \
+	  echo;                                                                \
+	  exit 1;                                                              \
+	}
 	$(E) "DYNASM    $@"
 	$(Q)$(DASM) -o $@ $(DASM_DASC)
 
@@ -341,10 +351,6 @@ lj_recdef.h: $(BUILDVM_T) $(LJLIB_C)
 	$(E) "BUILDVM   $@"
 	$(Q)$(BUILDVM_X) -m recdef -o $@ $(LJLIB_C)
 
-$(LIB_VMDEF): $(BUILDVM_T) $(LJLIB_C)
-	$(E) "BUILDVM   $@"
-	$(Q)$(BUILDVM_X) -m vmdef -o $(LIB_VMDEFP) $(LJLIB_C)
-
 lj_folddef.h: $(BUILDVM_T) lj_opt_fold.c
 	$(E) "BUILDVM   $@"
 	$(Q)$(BUILDVM_X) -m folddef -o $@ lj_opt_fold.c
@@ -365,7 +371,7 @@ lj_folddef.h: $(BUILDVM_T) lj_opt_fold.c
 
 $(LUAJIT_O):
 	$(E) "CC        $@"
-	$(Q)$(TARGET_STCC) $(TARGET_ACFLAGS) -c -o $@ $<
+	$(Q)$(TARGET_STCC) $(TARGET_ACFLAGS) -c -o $@ luajit.c
 
 $(HOST_O): %.o: %.c
 	$(E) "HOSTCC    $@"
@@ -389,6 +395,6 @@ $(LUAJIT_SO): $(LJVMCORE_O)
 $(LUAJIT_T): $(TARGET_O) $(LUAJIT_O) $(TARGET_DEP)
 	$(E) "LINK      $@"
 	$(Q)$(TARGET_LD) $(TARGET_ALDFLAGS) -o $@ $(LUAJIT_O) $(TARGET_O) $(TARGET_ALIBS)
-	$(E) "OK        Successfully built LuaJIT"
+	$(E) "OK        Successfully built RaptorJIT"
 
 ##############################################################################

src/host/.gitignore

@@ -1,3 +1,2 @@
 minilua
 buildvm
-buildvm_arch.h