Merge pull request #33 from docksal/develop

Release 1.6.0

.travis.yml

@@ -8,7 +8,7 @@ env:
     - REPO=docksal/ci-agent
 
 install:
-  - curl -fsSL get.docksal.io | sh
+  - curl -fsSL get.docksal.io | bash
   - fin version
   - fin sysinfo
 

README.md

@@ -59,15 +59,15 @@ If using `DOCKSAL_HOST_IP`, the agent will use `nip.io` for dynamic wildcard dom
 
 A base64 encoded private SSH key, used to access the remote Docksal host.
 
-`CI_SSH_KEY`
-
-A base64 encoded private SSH key, used by default for all hosts (set as `Host *` in `~/.ssh/config`).
-This key will be used to clone/push to repo, run commands over SSH on a remote deployment environment, etc.
-
 Note: `cat /path/to/<private_key_file> | base64` can be used to create a base64 encoded string from a private SSH key.
 
 ### Optional
 
+`CI_SSH_KEY`
+
+A base64 encoded private SSH key, used by default for all hosts (set as `Host *` in `~/.ssh/config`).
+This key will be used to clone/push to git, run commands over SSH on a remote deployment environment, etc.
+
 `DOCKSAL_DOMAIN`
 
 Can be used to set the base URL for sandbox builds (defaults to `DOCKSAL_HOST` if not set), individually from `DOCKSAL_HOST`.  
@@ -84,16 +84,16 @@ Defaults to `/home/ubuntu/builds`
 
 `REMOTE_CODEBASE_METHOD`
 
-Pick between `git` (default) and `rsync` for the codebase initialization method on the sandbox server.
+Pick between `git` and `rsync` (default) for the codebase initialization method on the sandbox server.
 
-The codebase is initialized on the sandbox server by the `build-init` command.
+The codebase is initialized on the sandbox server by the `sandbox-init` (or `build-init`) command.
 
 `git` - code is checkout on the sandbox server via git. Server must have access to checkout from the repo. 
 Any build settings and necessary code manipulations must happen on the sandbox server using `build-exec` commands.
 
-`rsync` - code is rsynced to the sandbox server from the build agent. You can perform necessary code adjustments in the 
-build agent after running `build-env` and before running `build-init`. The latter one will push the code to the sandbox 
-environment.
+`rsync` - code is rsync-ed to the sandbox server from the build agent. You can perform necessary code adjustments in the 
+build agent after running `build-env` and before running `sandbox-init` (or `build-init`), which pushes the code to the 
+sandbox server.
 
 `GITHUB_TOKEN` and `BITBUCKET_TOKEN`
 

base/bin/build-env

@@ -43,6 +43,11 @@ safe_string ()
 # Initial build environment configuration
 build_env ()
 {
+	# Drop variables with "null" values
+	# This allows, at the project level, unsetting build variables set at the org level
+	empty_vars="$(env | grep '=null$' | cut -d = -f 1)"
+	while read -r i; do unset ${i}; done <<< "${empty_vars}"
+
 	# Support for Bitbucket Pipelines
 	if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then
 		echo-debug "Detected Bitbucket Pipelines build environment"
@@ -69,13 +74,13 @@ build_env ()
 		export GIT_BRANCH_NAME="$CIRCLE_BRANCH"
 		export GIT_COMMIT_HASH="$CIRCLE_SHA1"
 
-		if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]]; then
+		if [[ "$CIRCLE_REPOSITORY_URL" == *"github.com"* ]]; then
 			export GIT_REPO_SERVICE="github"
 			# Figure out the pull request number
 			# Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds
 			export GIT_PR_NUMBER=${CIRCLE_PULL_REQUEST##*/}
 		fi
-		if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.org"* ]]; then
+		if [[ "$CIRCLE_REPOSITORY_URL" == *"bitbucket.org"* ]]; then
 			export GIT_REPO_SERVICE="bitbucket"
 			# Figure out the pull request number
 			# Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds
@@ -141,30 +146,53 @@ build_env ()
 	# Make sure domain name is lowercase
 	export DOCKSAL_DOMAIN="$(echo -n ${DOCKSAL_DOMAIN:-$DOCKSAL_HOST} | awk '{print tolower($0)}')"
 
-	# Use "flat" sub-domains (e.g. branch-project.example.com) and not multi-sub-domains (e.g. branch.project.example.com)
+	# Use "flat" sub-domains (e.g. branch--project.example.com) and not multi-sub-domains (e.g. branch.project.example.com)
 	# This allows using a single wildcard cert for the entire sandbox server.
 	# Note: A wildcard cert for "*.example.com", will only cover "sub-domain.example.dom", but not
 	# "www.sub-domain.example.com".
 	# NOTE: The length of any one label (sub-domain) in the domain name is limited to 63 octets (characters).
-	export DOMAIN="${BRANCH_NAME_SAFE}-${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}"
+	export DOMAIN="${BRANCH_NAME_SAFE}--${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}"
+
+	# Default to rsync for sandbox codebase initialization
+	export REMOTE_CODEBASE_METHOD="${REMOTE_CODEBASE_METHOD:-rsync}"
 }
 
 # Configure SSH keys
 # Note: CircleCI injects it's own key during checkout.
 # Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values.
 ssh_init ()
 {
+	mkdir -p $HOME/.ssh
+
+	# Default key used for all hosts
 	if [[ "$CI_SSH_KEY" != "" ]]; then
-		umask  077
 		echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa
 		chmod 0600 $HOME/.ssh/id_rsa
 	fi
 
+	# Docksal Sandbox server key
 	if [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]]; then
-		umask  077
 		echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa
 		chmod 0600 $HOME/.ssh/docksal_host_id_rsa
 	fi
+
+	# Initialize ssh-agent and load the default key ($HOME/.ssh/id_rsa)
+	# Check whether ssh-agent is configured
+	ssh-add -l &>/dev/null || ret=$?
+	# If ssh-agent is not configured, but config file exists, attempt to load agent settings from the file
+	if [[ "${ret}" == 2 ]] && [[ -f $HOME/.ssh/agent ]]; then
+		eval "$(<$HOME/.ssh/agent)" >/dev/null
+	fi
+	# Check whether ssh-agent is configured again
+	ssh-add -l &>/dev/null || ret=$?
+	# If the existing config was invalid, start a new agent and write new config
+	if [[ "${ret}" == 2 ]]; then
+		ssh-agent > $HOME/.ssh/agent
+		chmod 0600 $HOME/.ssh/agent
+		eval "$(<$HOME/.ssh/agent)" >/dev/null
+	fi
+	# Load default keys into the ssh-agent if available
+	ssh-add &>/dev/null || true
 }
 
 # Configure preferred git settings

base/bin/build-init

@@ -7,8 +7,8 @@ set -e # Abort if anything fails
 
 # Cleanup
 echo "Setting up remote build environment..."
-ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f 2>/dev/null) || true"
-ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR 2>/dev/null; mkdir -p $REMOTE_BUILD_DIR"
+ssh docker-host "(cd ${REMOTE_BUILD_DIR} 2>/dev/null && fin rm -f 2>/dev/null) || true"
+ssh docker-host "sudo rm -rf ${REMOTE_BUILD_DIR} 2>/dev/null; mkdir -p ${REMOTE_BUILD_DIR}"
 
 # Note: build-exec = ssh docker-host "cd $REMOTE_BUILD_DIR && ($@)"
 
@@ -20,25 +20,40 @@ if [[ "${REMOTE_CODEBASE_METHOD}" == "rsync" ]]; then
 else
 	# Checkout sources on the remote host
 	echo "Checking out codebase via git..."
-	build-exec "git clone --branch="$GIT_BRANCH_NAME" --depth 50 $GIT_REPO_URL . && git reset --hard $GIT_COMMIT_HASH && ls -la"
+	build-exec "git clone --branch="${GIT_BRANCH_NAME}" --depth 50 ${GIT_REPO_URL} . && git reset --hard ${GIT_COMMIT_HASH} && ls -la"
 fi
 
 # Configure sandbox settings
 echo "Configuring sandbox settings..."
-build-exec "echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env"
-build-exec "echo VIRTUAL_HOST=$DOMAIN | tee -a .docksal/docksal-local.env"
+build-exec "echo COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} | tee -a .docksal/docksal-local.env"
+build-exec "echo VIRTUAL_HOST=${DOMAIN} | tee -a .docksal/docksal-local.env"
 
 # Basic HTTP Auth
 if [[ "${HTTP_USER}" != "" ]] && [[ "${HTTP_PASS}" != "" ]]; then
 	echo "Configuring sandbox Basic HTTP Authentication..."
-	build-exec "echo APACHE_BASIC_AUTH_USER=$HTTP_USER | tee -a .docksal/docksal-local.env"
-	build-exec "echo APACHE_BASIC_AUTH_PASS=$HTTP_PASS | tee -a .docksal/docksal-local.env"
+	build-exec "echo APACHE_BASIC_AUTH_USER=${HTTP_USER} | tee -a .docksal/docksal-local.env"
+	build-exec "echo APACHE_BASIC_AUTH_PASS=${HTTP_PASS} | tee -a .docksal/docksal-local.env"
 fi
 
+# Permanent environment switch
+if [[ "${SANDBOX_PERMANENT}" != "" ]]; then
+	echo "Setting sandbox as permanent..."
+	build-exec "echo SANDBOX_PERMANENT=${SANDBOX_PERMANENT} | tee -a .docksal/docksal-local.env"
+fi
+
+# Pass CI_SSH_KEY to sandbox
+# Note the key is passed as SECRET_SSH_PRIVATE_KEY, which docksal/cli reads, decodes and stores as ~/.ssh/id_rsa
+# Disabled for now. This may be a security concern, if a single shared machine-user SSH key is used across multiple projects.
+# TODO: Load the key into the docksal/ssh-agent service on the sandbox server instead.
+#if [[ "${CI_SSH_KEY}" != "" ]]; then
+#	echo "Passing CI_SSH_KEY to sandbox..."
+#	build-exec "echo SECRET_SSH_PRIVATE_KEY=\"${CI_SSH_KEY}\" | tee -a .docksal/docksal-local.env >/dev/null"
+#fi
+
 # Pass build secrets to sandbox
 # A "secret" is any environment variable that starts with "SECRET_"
-secrets="$(env | grep 'SECRET_')" || true
-if [[ "$secrets" != "" ]]; then
+secrets="$(env | grep '^SECRET_')" || true
+if [[ "${secrets}" != "" ]]; then
 	echo "Passing build secrets to sandbox..."
-	build-exec "echo '$secrets' | tee -a .docksal/docksal-local.env >/dev/null"
+	build-exec "echo '${secrets}' | tee -a .docksal/docksal-local.env >/dev/null"
 fi

base/bin/sandbox-init

@@ -3,9 +3,9 @@
 # This script initializes a sandbox environment using the default settings
 
 echo "Initializing codebase and settings on the sandbox server..."
-build-init
-[[ $? == 0 ]] && build-notify pending || build-notify failure
+build-init; ret=$?
+[[ ${ret} == 0 ]] && build-notify pending || { build-notify failure; exit ${ret}; }
 
 echo "Initializing sandbox via 'fin init'..."
-build-exec "fin init"
-[[ $? == 0 ]] && build-notify success || build-notify failure
+build-exec "fin init"; ret=$?
+[[ ${ret} == 0 ]] && build-notify success || { build-notify failure; exit ${ret}; }

base/config/.ssh/config

@@ -19,3 +19,6 @@ Host docker-host
 	LogLevel ERROR
 	IdentityFile ~/.ssh/docksal_host_id_rsa
 	ControlPath ~/.ssh/docksal_host.ctl
+	# Enabled ssh-agent forwarding
+	# This passes identities loaded into the ssh-agent in the ci-agent to the sandbox server
+	ForwardAgent yes

tests/base.bats

@@ -39,3 +39,51 @@ teardown() {
 	### Cleanup ###
 	make clean
 }
+
+@test "Git settings" {
+	[[ $SKIP == 1 ]] && skip
+
+	### Setup ###
+	make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash"'
+
+	### Tests ###
+	# Check git settings were applied
+	run make exec COMMAND="build-env"
+	run make exec COMMAND="git config --get --global user.email"
+	[[ "$status" == 0 ]]
+	echo "$output" | grep "git@example.com"
+	unset output
+
+	run make exec COMMAND="build-env"
+	run make exec COMMAND="git config --get --global user.name"
+	[[ "$status" == 0 ]]
+	echo "$output" | grep "Docksal CLI"
+	unset output
+
+	### Cleanup ###
+	make clean
+}
+
+@test "Check SSH agent and keys" {
+	[[ $SKIP == 1 ]] && skip
+
+	### Setup ###
+	CI_SSH_KEY="LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUkxOElNMTZEMFVsS0U0VHVYeU1iQ3NEb3VHWU9TZC85SkJmSTcrenFCQ1JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1BSR2RQOUpSTmljbVQ1RjR1WFNEakV2TXFUczAxaVVOTXprTXAzUVdSM3hScWp5VFlYdAp0R1hRNE5BVWlxWEtlMnNaN0NZMmxqeGNrTUJCamU2OEhBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
+	make start -e ENV="-e GIT_USER_EMAIL='git@example.com' -e GIT_USER_NAME='Docksal CLI' -e GIT_REPO_URL='test-repo-url' -e GIT_BRANCH_NAME='test-branch-name' -e GIT_COMMIT_HASH='test-commit-hash' -e CI_SSH_KEY='${CI_SSH_KEY}'"
+	make exec COMMAND="build-env"
+
+	### Tests ###
+
+	# Check private SSH key
+	run make exec COMMAND='bash -lc "echo \$$CI_SSH_KEY | base64 -d | diff \$$HOME/.ssh/id_rsa -"'
+	[[ "$status" == 0 ]]
+	unset output
+
+	# Check ssh-agent
+	run make exec COMMAND='bash -lc "source build-env; ssh-add -l"'
+	[[ "$status" == 0 ]]
+	unset output
+
+	### Cleanup ###
+	make clean
+}