Compile zulu source and package files

A number of changes made to the way Zulu and its packages are loaded in
order to dramatically increase startup performance.

* Source is now moved into `src` directory, and the script `build.zsh` has been added to the repository, which concatenates all the source files into a single script with comments removed for speed.
* The `self-update` command has been modified to run the `build.zsh` script once it has finished downloading updates.
* Adds a new `compile` command, which runs `zcompile` on the Zulu core, all files in `~/.zulu/{bin,share,init}`, the completion cache and the following files in the user's `$ZDOTDIR`:
  * `.zlogin`
  * `.zshenv`
  * `.zprofile`
  * `.zshrc`
  * `.zlogout`
  The end result is a massive increase in startup speed.
* `zulu compile` is run in the background to compile sources on startup. A new `--no-compile` option has been added to `zulu init` to avoid this.

.gitignore

@@ -0,0 +1,11 @@
+# Ignore files generated by nested Zulu install
+tests/_support/.config/*
+tests/_support/.zulu/*
+tests/_support/zulu-install.zsh
+
+# Ignore output
+tests/_output/*
+!tests/_output/.gitkeep
+
+/zulu
+*.zwc

.guardian.yml

@@ -0,0 +1,5 @@
+files: ./tests/commands/**/*
+run: zunit "%file%"
+---
+files: ./src/**/*.zsh
+run: ./build.zsh

build.zsh

@@ -0,0 +1,23 @@
+#!/usr/bin/env zsh
+
+# Clear the file to start with
+cat /dev/null > zulu
+
+# Start with the shebang
+echo "#!/usr/bin/env zsh\n" >> zulu
+
+# We need to do some fancy globbing
+setopt EXTENDED_GLOB
+
+# Print each of the source files into the target, removing any comments
+# and blank lines from the compiled executable
+cat src/**/(^zulu).zsh | grep -v -E '^(\s*#.*[^"]|\s*)$' >> zulu
+
+# Print the main command last
+cat src/zulu.zsh | grep -v -E '^(\s*#.*[^"]|\s*)$' >> zulu
+
+# Make sure the file is executable
+chmod u+x zulu
+
+# Let the user know we're finished
+echo "\033[0;32m✔\033[0;m zulu built successfully"

commands/alias → src/commands/alias.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###
@@ -16,7 +14,7 @@ function _zulu_alias_usage() {
 ###
 # Add an alias
 ###
-_zulu_alias_add() {
+function _zulu_alias_add() {
   local existing alias cmd
 
   alias="$1"
@@ -31,13 +29,13 @@ _zulu_alias_add() {
   echo "alias $alias='$cmd'" >> $aliasfile
 
   zulu alias load
-  return
+  echo "$(_zulu_color green '✔') Alias '$alias' added"
 }
 
 ###
 # Remove an alias
 ###
-_zulu_alias_rm() {
+function _zulu_alias_rm() {
   local existing alias
 
   alias="$1"
@@ -50,14 +48,15 @@ _zulu_alias_rm() {
 
   echo "$(cat $aliasfile | grep -v "alias $alias=")" >! $aliasfile
   unalias $alias
+
   zulu alias load
-  return
+  echo "$(_zulu_color green '✔') Alias '$alias' removed"
 }
 
 ###
 # Load aliases
 ###
-_zulu_alias_load() {
+function _zulu_alias_load() {
   source $aliasfile
 }
 

commands/bundle → src/commands/bundle.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Print usage information
 ###

commands/cdpath → src/commands/cdpath.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###
@@ -108,13 +106,6 @@ function _zulu_cdpath_rm() {
 function _zulu_cdpath_store() {
   local separator out
 
-  # Check that we have all the parameters we need. This will only happen if
-  # _zulu_cdpath_store is executed internally.
-  if [[ "$items" = "" || "$pathfile" = "" ]]; then
-    echo 'Missing parameters. Was _zulu_cdpath_store called from within the add or remove functions?'
-    return 1
-  fi
-
   # Separate the array by newlines, and print the contents to the pathfile
   separator=$'\n'
   local IFS="$separator"; out="${items[*]/#/${separator}}"

src/commands/compile.zsh

@@ -0,0 +1,90 @@
+#!/usr/bin/env zsh
+
+###
+# Output usage information and exit
+###
+function _zulu_compile_usage() {
+  echo '\033[0;32mUsage:\033[0;m'
+  echo '  zulu_compile [options]'
+}
+
+###
+# Resolve symbolic links to a file, a compare it's last-modified date
+# with the compiled version, recompiling if needed
+###
+function _zulu_compile_if_needed() {
+  local file="$1" follow_symlinks
+
+  zparseopts -D \
+    f=follow_symlinks -follow-symlinks=follow_symlinks
+
+  # We can't compile files that do not exist
+  [[ ! -e $file ]] && return
+
+  # Resolve symlinks if necessary
+  [[ -n $follow_symlinks && -L $file ]] && file=$(readlink $file)
+
+  # Check if the file is newer than it's compiled version,
+  # and recompile if necessary
+  if [[ -s ${file} && ( ! -s ${file}.zwc || ${file} -nt ${file}.zwc) ]]; then
+    zcompile ${file}
+  fi
+}
+
+###
+# The main zulu_compile process
+###
+(( $+functions[_zulu_compile] )) || function _zulu_compile() {
+  local base=${ZULU_DIR:-"${ZDOTDIR:-$HOME}/.zulu"}
+  local config=${ZULU_CONFIG_DIR:-"${ZDOTDIR:-$HOME}/.config/zulu"}
+  local help version
+
+  zparseopts -D \
+    h=help -help=help
+
+  if [[ -n $help ]]; then
+    _zulu_compile_usage
+    exit
+  fi
+
+  setopt EXTENDED_GLOB
+  # A list of glob paths pointing to files to be compiled
+  local -a compile_targets; compile_targets=(
+    # Zulu's core
+    ${base}/core/zulu
+
+    # Files linked by packages
+    ${base}/share/**/*^(*.zwc)(#q@)
+    ${base}/bin/**/*^(*.zwc)(#q@)
+
+    # Completion dump
+    ${ZDOTDIR:-${HOME}}/.zcomp^(*.zwc)(.)
+
+    # User env files
+    ${ZDOTDIR:-${HOME}}/.zshenv
+    ${ZDOTDIR:-${HOME}}/.zlogin
+    ${ZDOTDIR:-${HOME}}/.zprofile
+    ${ZDOTDIR:-${HOME}}/.zshrc
+    ${ZDOTDIR:-${HOME}}/.zlogout
+  )
+
+  # A second list of compile targets. These files have their symlinks resolved
+  # before they are sourced, so we need to follow the symlink before compiling,
+  # to ensure the compiled version is picked up
+  local -a linked_compile_targets; linked_compile_targets=(
+    # Initialisation scripts for packages
+    ${base}/init/**/*^(*.zwc)(#q@)
+  )
+
+  # Loop through each of the files marked for compilation, and compile
+  # them if necessary
+  for file in ${(@f)compile_targets}; do
+    _zulu_compile_if_needed $file
+  done
+
+  # Loop through each of the files marked for compilation, follow their
+  # symlinks, and compile them if necessary
+  for file in ${(@f)linked_compile_targets}; do
+    _zulu_compile_if_needed --follow-symlinks $file
+  done
+}

commands/fpath → src/commands/fpath.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###
@@ -108,13 +106,6 @@ function _zulu_fpath_rm() {
 function _zulu_fpath_store() {
   local separator out
 
-  # Check that we have all the parameters we need. This will only happen if
-  # _zulu_fpath_store is executed internally.
-  if [[ "$items" = "" || "$pathfile" = "" ]]; then
-    echo 'Missing parameters. Was _zulu_fpath_store called from within the add or remove functions?'
-    return 1
-  fi
-
   # Separate the array by newlines, and print the contents to the pathfile
   separator=$'\n'
   local IFS="$separator"; out="${items[*]/#/${separator}}"

commands/func → src/commands/func.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###

commands/info → src/commands/info.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###

commands/init → src/commands/init.zsh

@@ -1,12 +1,11 @@
-#!/usr/bin/env zsh
-
 function _zulu_init_usage() {
   echo $(_zulu_color yellow "Usage:")
   echo "  zulu init [options]"
   echo
   echo $(_zulu_color yellow "Options:")
   echo "  -c, --check-for-update   Check for updates on startup"
   echo "  -h, --help               Output this help text and exit"
+  echo "  -n, --no-compile         Skip compilation of scripts on startup"
 }
 
 function _zulu_init_setup_completion() {
@@ -557,21 +556,13 @@ function _zulu_check_for_update() {
   fi
 }
 
-###
-# Load the zulu commands
-###
-function _zulu_load_commands() {
-  for cmd in $(find "$base/core/commands"); do
-    source $cmd
-  done
-}
-
 ###
 # Source init scripts for installed packages
 ###
 function _zulu_load_packages() {
   # Source files in the init directory
-  for f in $(find "$base/init"); do
+  setopt EXTENDED_GLOB
+  for f in ${base}/init/**/*^(*.zwc)(#q@N); do
     if [[ -L $f ]]; then
       source $(readlink $f)
     else
@@ -586,33 +577,25 @@ function _zulu_load_packages() {
 function _zulu_init() {
   local base=${ZULU_DIR:-"${ZDOTDIR:-$HOME}/.zulu"}
   local config=${ZULU_CONFIG_DIR:-"${ZDOTDIR:-$HOME}/.config/zulu"}
-  local help check_for_update
-
-  # Ensure path arrays do not contain duplicates.
-  typeset -gU cdpath fpath mailpath path
+  local help check_for_update no_compile
 
   # Parse CLI options
   zparseopts -D \
     h=help -help=help \
-    c=check_for_update -check-for-update=check_for_update
+    c=check_for_update -check-for-update=check_for_update \
+    n=no_compile -no-compile=no_compile
 
   if [[ -n $help ]]; then
     _zulu_init_usage
     return
   fi
 
-  # Load zulu internal commands
-  _zulu_load_commands
-
   # Populate paths
   zulu path reset
   zulu fpath reset
   zulu cdpath reset
   zulu manpath reset
 
-  # Source helper functions
-  source "$base/core/helpers"
-
   # Set up the environment
   _zulu_init_setup_key_bindings
   _zulu_init_setup_completion
@@ -635,5 +618,11 @@ function _zulu_init() {
     prompt $theme
   fi
 
+  if [[ -z $no_compile ]]; then
+    {
+      zulu compile
+    } >/dev/null 2>&1 &!
+  fi
+
   [[ -n $check_for_update ]] && _zulu_check_for_update
 }

commands/install → src/commands/install.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###

commands/link → src/commands/link.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Print usage information
 ###
@@ -81,17 +79,20 @@ function _zulu_link() {
 
     # Loop through each of the values in the array, the key is a file within
     # the package, the value is the name of a symlink to create in the directory
-    for file link (${(@kv)files}) ln -s $root/${~file} $base/$dir/$link
-
-    # Make sure that commands to be included in bin are executable
-    if [[ "$dir" = "bin" ]]; then
-      for file link ("${(@kv)files}") chmod u+x "$root/$file"
-    fi
-
-    # Source init scripts
-    if [[ "$dir" = "init" ]]; then
-      for file link ("${(@kv)files}") source $(readlink "$base/init/$link")
-    fi
+    for file link in "${(@kv)files}"; do
+      # Create a symlink to the file, filtering out .zwc files
+      ln -s $root/${~file} $base/$dir/$link
+
+      # Make sure that commands to be included in bin are executable
+      if [[ "$dir" = "bin" ]]; then
+        chmod u+x "$root/$file"
+      fi
+
+      # Source init scripts
+      if [[ "$dir" = "init" ]]; then
+        source $(readlink "$base/init/$link")
+      fi
+    done
   done
 
   package_type=$(jsonval $json 'type')

commands/list → src/commands/list.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Print usage information
 ###

commands/manpath → src/commands/manpath.zsh

@@ -1,5 +1,3 @@
-#!/usr/bin/env zsh
-
 ###
 # Output usage information
 ###