Merge branch 'raptorjit/master' into auditlog

.travis.yml

@@ -1,9 +1,10 @@
 language: nix
 sudo: false
 env:
-  - test=test-O3
-  - test=test-O2
-  - test=test-O1
-  - test=test-nojit
+  - nix-build -A test-O3
+  - nix-build -A test-O2
+  - nix-build -A test-O1
+  - nix-build -A test-nojit
+  - nix-build -A check-generated-code --arg check true
 script:
-  - nix-build -A $test
+  - $test

Makefile

@@ -1,5 +1,5 @@
 ##############################################################################
-# LuaJIT top level Makefile for installation. Requires GNU Make.
+# RaptorJIT top level Makefile for installation. Requires GNU Make.
 #
 # Please read doc/install.html before changing any variables!
 #
@@ -13,10 +13,10 @@
 # Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
 ##############################################################################
 
-MAJVER=  2
-MINVER=  1
+MAJVER=  1
+MINVER=  0
 RELVER=  0
-PREREL=  -beta2
+PREREL=  -alpha1
 VERSION= $(MAJVER).$(MINVER).$(RELVER)$(PREREL)
 ABIVER=  5.1
 
@@ -109,12 +109,12 @@ endif
 INSTALL_DEP= src/luajit
 
 default all $(INSTALL_DEP):
-	@echo "==== Building LuaJIT $(VERSION) ===="
+	@echo "==== Building RaptorJIT $(VERSION) ===="
 	$(MAKE) -C src
-	@echo "==== Successfully built LuaJIT $(VERSION) ===="
+	@echo "==== Successfully built RaptorJIT $(VERSION) ===="
 
 install: $(INSTALL_DEP)
-	@echo "==== Installing LuaJIT $(VERSION) to $(PREFIX) ===="
+	@echo "==== Installing RaptorJIT $(VERSION) to $(PREFIX) ===="
 	$(MKDIR) $(INSTALL_DIRS)
 	cd src && $(INSTALL_X) $(FILE_T) $(INSTALL_T)
 	cd src && test -f $(FILE_A) && $(INSTALL_F) $(FILE_A) $(INSTALL_STATIC) || :
@@ -130,7 +130,7 @@ install: $(INSTALL_DEP)
 	  $(RM) $(FILE_PC).tmp
 	cd src && $(INSTALL_F) $(FILES_INC) $(INSTALL_INC)
 	cd src/jit && $(INSTALL_F) $(FILES_JITLIB) $(INSTALL_JITLIB)
-	@echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ===="
+	@echo "==== Successfully installed RaptorJIT $(VERSION) to $(PREFIX) ===="
 	@echo ""
 	@echo "Note: the development releases deliberately do NOT install a symlink for luajit"
 	@echo "You can do this now by running this command (with sudo):"
@@ -140,7 +140,7 @@ install: $(INSTALL_DEP)
 
 
 uninstall:
-	@echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ===="
+	@echo "==== Uninstalling RaptorJIT $(VERSION) from $(PREFIX) ===="
 	$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC)
 	for file in $(FILES_JITLIB); do \
 	  $(UNINSTALL) $(INSTALL_JITLIB)/$$file; \
@@ -150,13 +150,16 @@ uninstall:
 	  done
 	$(LDCONFIG) $(INSTALL_LIB)
 	$(RMDIR) $(UNINSTALL_DIRS) || :
-	@echo "==== Successfully uninstalled LuaJIT $(VERSION) from $(PREFIX) ===="
+	@echo "==== Successfully uninstalled RaptorJIT $(VERSION) from $(PREFIX) ===="
 
 ##############################################################################
 
 clean:
 	$(MAKE) -C src clean
 
+bootstrapclean:
+	$(MAKE) -C src bootstrapclean
+
 .PHONY: all install clean
 
 ##############################################################################

README.md

@@ -68,16 +68,31 @@ details in discussions then these are the places to look for answers.
 
 The [AnandTech review of the Haswell microarchitecture](http://www.anandtech.com/show/6355/intels-haswell-architecture) is also excellent lighter reading.
 
-### Compilation
+### Compilation for users
 
-RaptorJIT uses [nix](http://nixos.org/nix/) to define a reproducible
-build environment that includes Clang for C and LuaJIT 2.0 for
-bootstrapping (see [default.nix](default.nix)). The recommended way to
-build RaptorJIT is with nix, which provides the dependencies
-automatically, but you can build manually if you prefer.
+Simple build:
 
-Building with nix will be slow the first time due to downloading
-toolchains and related dependencies. This is all cached for future
+```shell
+$ make  # requires LuaJIT (2.0 or 2.1) to run DynASM
+```
+
+Alternative if you don't have LuaJIT available and you are building a
+pristine copy from the master branch:
+
+```shell
+$ make reusevm  # Reuse reference copy of the generated VM code
+$ make          # Does not require LuaJIT now
+```
+
+### Compilation for VM hackers
+
+RaptorJIT uses [Nix](http://nixos.org/nix/) to provide a reference
+build environment. You can use Nix to build/test/benchmark RaptorJIT
+with suitable versions of all dependencies provided.
+
+Note: Building with nix will be slow the first time because it
+downloads the exact reference versions of the toolchain (clang, etc)
+and all dependencies (glibc, etc). This is all cached for future
 builds.
 
 #### Build with nix
@@ -116,6 +131,40 @@ $ make
 
 ... but make sure you have at least `make`, `clang`, and `luajit` in your `$PATH`.
 
+### Run the benchmarks
+
+Nix can also run the full benchmark suite and generate visualizations
+with R/ggplot2.
+
+The simplest incantation tests one branch:
+
+```shell
+$ nix-build testsuite/bench --arg Asrc ./.   # note: ./. means ./
+```
+
+You can also test several branches (A-E), give them names, specify
+command-line arguments, say how many tests to run, and allow parallel
+execution:
+
+```shell
+# Run the benchmarks and create result visualizations result/
+$ nix-build testsuite/bench                     \
+            --arg    Asrc ~/git/raptorjit       \
+            --argstr Aname master               \
+            --arg    Bsrc ~/git/raptorjit-hack  \
+            --argstr Bname hacked               \
+            --arg    Csrc ~/git/raptorjit-hack2 \
+            --argstr Cname hacked-O1            \
+            --argstr Cargs -O1                  \
+            --arg    runs 100                   \
+            -j 5           # Run up to 5 tests in parallel
+```
+
+If you are using a distributed nix environment such
+as [Hydra](https://nixos.org/hydra/) then the tests can be
+automatically parallelized and distributed across a suitable build
+farm.
+
 ### Quotes
 
 Here are some borrowed words to put this branch into context:

check-generated-code.nix

@@ -0,0 +1,31 @@
+# Check that generated sources match the repo version.
+{ pkgs, raptorjit }:
+with pkgs; with lib;
+
+# Generated files that are kept in tree.
+let generatedFiles =
+  "lj_bcdef.h lj_ffdef.h lj_libdef.h lj_recdef.h lj_folddef.h host/buildvm_arch.h";
+in
+
+overrideDerivation raptorjit (as:
+  {
+    preBuild = ''
+      pushd src
+      mkdir old
+      for f in ${generatedFiles}; do
+        cp $f old/
+      done
+      popd
+    '' + as.preBuild;
+    checkPhase = ''
+      pushd src
+      mkdir new
+      for f in ${generatedFiles}; do
+        cp $f new/
+      done
+      echo "Checking that in-tree generated VM code is up-to-date..."
+      diff -u old new || (echo "Error: Stale generated code"; exit 1)
+      popd
+    '';
+    doCheck = true;
+  })

default.nix

@@ -9,12 +9,14 @@
 
 { pkgs ? (import ./pkgs.nix) {}
 , source ? pkgs.lib.cleanSource ./.
-, version ? "dev" }:
+, version ? "dev"
+, check ? false }:
 
 let
-  callPackage = (pkgs.lib.callPackageWith { inherit pkgs; inherit source; inherit version; });
+  callPackage = (pkgs.lib.callPackageWith { inherit pkgs source version; });
   raptorjit = (callPackage ./raptorjit.nix {});
-  test = name: args: (callPackage ./test.nix { inherit raptorjit; inherit name; inherit args; });
+  test = name: args: (callPackage ./test.nix { inherit raptorjit name args; });
+  check-generated-code = (callPackage ./check-generated-code.nix { inherit raptorjit; });
 in
 
 # Build RaptorJIT and run mulitple test suites.
@@ -24,5 +26,6 @@ in
   test-O2    = test "O2"    "-O2";
   test-O1    = test "O1"    "-O1";
   test-nojit = test "nojit" "-joff";
-}
+} //
+(if check then { inherit check-generated-code; } else {})
 

src/.gitignore

@@ -1,7 +1 @@
 luajit
-lj_bcdef.h
-lj_ffdef.h
-lj_libdef.h
-lj_recdef.h
-lj_folddef.h
-lj_vm.[sS]