From 7196199f56d5c3fba87555fe1dc5885f6bd67e82 Mon Sep 17 00:00:00 2001 From: sergey-zabolotny <37826765+sergey-zabolotny@users.noreply.github.com> Date: Tue, 16 Oct 2018 19:57:04 +0300 Subject: [PATCH 01/16] Add tests for git settings and ssh keys (#32) * Add git and ssh keys tests * Execute docksal installation using bash --- .travis.yml | 2 +- tests/base.bats | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01ea10f..2e4e090 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: - REPO=docksal/ci-agent install: - - curl -fsSL get.docksal.io | sh + - curl -fsSL get.docksal.io | bash - fin version - fin sysinfo diff --git a/tests/base.bats b/tests/base.bats index 3a3e402..678a6e3 100755 --- a/tests/base.bats +++ b/tests/base.bats @@ -39,3 +39,44 @@ teardown() { ### Cleanup ### make clean } + +@test "Git settings" { + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash"' + + ### Tests ### + # Check git settings were applied + run make exec COMMAND="build-env" + run make exec COMMAND="git config --get --global user.email" + [[ "$status" == 0 ]] + echo "$output" | grep "git@example.com" + unset output + + run make exec COMMAND="build-env" + run make exec COMMAND="git config --get --global user.name" + [[ "$status" == 0 ]] + echo "$output" | grep "Docksal CLI" + unset output + + ### Cleanup ### + make clean +} + +@test "Check SSH keys" { + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash" -e CI_SSH_KEY="dGVzdC1zc2gta2V5Cg=="' + + ### Tests ### + # Check private SSH key + run make exec COMMAND="build-env" + run make exec COMMAND='bash -lc "echo \$$CI_SSH_KEY | base64 -d | diff \$$HOME/.ssh/id_rsa -"' + [[ "$status" == 0 ]] + unset output + + ### Cleanup ### + make clean +} From edbcef6652e55afb29366c1d9c4fe27e90cd739a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 23 Oct 2018 14:52:58 -0700 Subject: [PATCH 02/16] Fail build if any of the steps in sandbox-init fail --- base/bin/sandbox-init | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/bin/sandbox-init b/base/bin/sandbox-init index 95505b5..4807171 100755 --- a/base/bin/sandbox-init +++ b/base/bin/sandbox-init @@ -3,9 +3,9 @@ # This script initializes a sandbox environment using the default settings echo "Initializing codebase and settings on the sandbox server..." -build-init -[[ $? == 0 ]] && build-notify pending || build-notify failure +build-init; ret=$? +[[ ${ret} == 0 ]] && build-notify pending || { build-notify failure; exit ${ret}; } echo "Initializing sandbox via 'fin init'..." -build-exec "fin init" -[[ $? == 0 ]] && build-notify success || build-notify failure +build-exec "fin init"; ret=$? +[[ ${ret} == 0 ]] && build-notify success || { build-notify failure; exit ${ret}; } From b9e96325345ac7545acb8647c9c76333cff60cb4 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 23 Oct 2018 16:10:13 -0700 Subject: [PATCH 03/16] Enabled ssh-agent forwarding --- base/config/.ssh/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/config/.ssh/config b/base/config/.ssh/config index b068703..922a34f 100644 --- a/base/config/.ssh/config +++ b/base/config/.ssh/config @@ -19,3 +19,6 @@ Host docker-host LogLevel ERROR IdentityFile ~/.ssh/docksal_host_id_rsa ControlPath ~/.ssh/docksal_host.ctl + # Enabled ssh-agent forwarding + # This passes identities loaded into the ssh-agent in the ci-agent to the sandbox server + ForwardAgent yes From 5a99da485dbb198166b67f23ec92b84c2e7ba415 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 31 Oct 2018 11:20:16 -0700 Subject: [PATCH 04/16] Drop variables with "null" values in build-env This allows, at the project level, unsetting build variables set at the org level --- base/bin/build-env | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index fb6b2f7..b7728f2 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -43,6 +43,11 @@ safe_string () # Initial build environment configuration build_env () { + # Drop variables with "null" values + # This allows, at the project level, unsetting build variables set at the org level + empty_vars="$(env | grep '=null$' | cut -d = -f 1)" + while read -r i; do unset ${i}; done <<< "${empty_vars}" + # Support for Bitbucket Pipelines if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo-debug "Detected Bitbucket Pipelines build environment" @@ -69,13 +74,13 @@ build_env () export GIT_BRANCH_NAME="$CIRCLE_BRANCH" export GIT_COMMIT_HASH="$CIRCLE_SHA1" - if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]]; then + if [[ "$CIRCLE_REPOSITORY_URL" == *"github.com"* ]]; then export GIT_REPO_SERVICE="github" # Figure out the pull request number # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds export GIT_PR_NUMBER=${CIRCLE_PULL_REQUEST##*/} fi - if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.org"* ]]; then + if [[ "$CIRCLE_REPOSITORY_URL" == *"bitbucket.org"* ]]; then export GIT_REPO_SERVICE="bitbucket" # Figure out the pull request number # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds From 9372cbc3c83ba2ad30ac2f7231cc5f6af952ec0c Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 1 Nov 2018 09:24:21 -0700 Subject: [PATCH 05/16] Use a double hyphen to separate branch from project in the domain name Note: thanks to Netlify for this simple and brilliant idea :) --- base/bin/build-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index b7728f2..628df4f 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -146,12 +146,12 @@ build_env () # Make sure domain name is lowercase export DOCKSAL_DOMAIN="$(echo -n ${DOCKSAL_DOMAIN:-$DOCKSAL_HOST} | awk '{print tolower($0)}')" - # Use "flat" sub-domains (e.g. branch-project.example.com) and not multi-sub-domains (e.g. branch.project.example.com) + # Use "flat" sub-domains (e.g. branch--project.example.com) and not multi-sub-domains (e.g. branch.project.example.com) # This allows using a single wildcard cert for the entire sandbox server. # Note: A wildcard cert for "*.example.com", will only cover "sub-domain.example.dom", but not # "www.sub-domain.example.com". # NOTE: The length of any one label (sub-domain) in the domain name is limited to 63 octets (characters). - export DOMAIN="${BRANCH_NAME_SAFE}-${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" + export DOMAIN="${BRANCH_NAME_SAFE}--${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" } # Configure SSH keys From 62ff01852ad282b9944459292605a1d9b50c9adf Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 14:00:59 -0700 Subject: [PATCH 06/16] Default to using rsync for codebase initialization on the server --- README.md | 10 +++++----- base/bin/build-env | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fab7ed5..37efa9f 100644 --- a/README.md +++ b/README.md @@ -84,16 +84,16 @@ Defaults to `/home/ubuntu/builds` `REMOTE_CODEBASE_METHOD` -Pick between `git` (default) and `rsync` for the codebase initialization method on the sandbox server. +Pick between `git` and `rsync` (default) for the codebase initialization method on the sandbox server. -The codebase is initialized on the sandbox server by the `build-init` command. +The codebase is initialized on the sandbox server by the `sandbox-init` (or `build-init`) command. `git` - code is checkout on the sandbox server via git. Server must have access to checkout from the repo. Any build settings and necessary code manipulations must happen on the sandbox server using `build-exec` commands. -`rsync` - code is rsynced to the sandbox server from the build agent. You can perform necessary code adjustments in the -build agent after running `build-env` and before running `build-init`. The latter one will push the code to the sandbox -environment. +`rsync` - code is rsync-ed to the sandbox server from the build agent. You can perform necessary code adjustments in the +build agent after running `build-env` and before running `sandbox-init` (or `build-init`), which pushes the code to the +sandbox server. `GITHUB_TOKEN` and `BITBUCKET_TOKEN` diff --git a/base/bin/build-env b/base/bin/build-env index 628df4f..6a0d68a 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -152,6 +152,9 @@ build_env () # "www.sub-domain.example.com". # NOTE: The length of any one label (sub-domain) in the domain name is limited to 63 octets (characters). export DOMAIN="${BRANCH_NAME_SAFE}--${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" + + # Default to rsync for sandbox codebase initialization + export REMOTE_CODEBASE_METHOD="${REMOTE_CODEBASE_METHOD:-rsync}" } # Configure SSH keys From c91fc02dcb2dae271759ed907c2c82ec526bb83a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 14:02:06 -0700 Subject: [PATCH 07/16] Mute cd command in build-init --- base/bin/build-init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-init b/base/bin/build-init index 6574d2e..0540693 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -7,8 +7,8 @@ set -e # Abort if anything fails # Cleanup echo "Setting up remote build environment..." -ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f 2>/dev/null) || true" -ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR 2>/dev/null; mkdir -p $REMOTE_BUILD_DIR" +ssh docker-host "(cd ${REMOTE_BUILD_DIR} 2>/dev/null && fin rm -f 2>/dev/null) || true" +ssh docker-host "sudo rm -rf ${REMOTE_BUILD_DIR} 2>/dev/null; mkdir -p ${REMOTE_BUILD_DIR}" # Note: build-exec = ssh docker-host "cd $REMOTE_BUILD_DIR && ($@)" From 66fc2ad6975e255887afde6a22ad946e414bdc97 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 14:03:26 -0700 Subject: [PATCH 08/16] Quotes and curly brackets around variables in built-init --- base/bin/build-init | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/base/bin/build-init b/base/bin/build-init index 0540693..f220bfc 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -20,25 +20,31 @@ if [[ "${REMOTE_CODEBASE_METHOD}" == "rsync" ]]; then else # Checkout sources on the remote host echo "Checking out codebase via git..." - build-exec "git clone --branch="$GIT_BRANCH_NAME" --depth 50 $GIT_REPO_URL . && git reset --hard $GIT_COMMIT_HASH && ls -la" + build-exec "git clone --branch="${GIT_BRANCH_NAME}" --depth 50 ${GIT_REPO_URL} . && git reset --hard ${GIT_COMMIT_HASH} && ls -la" fi # Configure sandbox settings echo "Configuring sandbox settings..." -build-exec "echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env" -build-exec "echo VIRTUAL_HOST=$DOMAIN | tee -a .docksal/docksal-local.env" +build-exec "echo COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} | tee -a .docksal/docksal-local.env" +build-exec "echo VIRTUAL_HOST=${DOMAIN} | tee -a .docksal/docksal-local.env" # Basic HTTP Auth if [[ "${HTTP_USER}" != "" ]] && [[ "${HTTP_PASS}" != "" ]]; then echo "Configuring sandbox Basic HTTP Authentication..." - build-exec "echo APACHE_BASIC_AUTH_USER=$HTTP_USER | tee -a .docksal/docksal-local.env" - build-exec "echo APACHE_BASIC_AUTH_PASS=$HTTP_PASS | tee -a .docksal/docksal-local.env" + build-exec "echo APACHE_BASIC_AUTH_USER=${HTTP_USER} | tee -a .docksal/docksal-local.env" + build-exec "echo APACHE_BASIC_AUTH_PASS=${HTTP_PASS} | tee -a .docksal/docksal-local.env" +fi + +# Permanent environment switch +if [[ "${SANDBOX_PERMANENT}" != "" ]]; then + echo "Setting sandbox as permanent..." + build-exec "echo SANDBOX_PERMANENT=${SANDBOX_PERMANENT} | tee -a .docksal/docksal-local.env" fi # Pass build secrets to sandbox # A "secret" is any environment variable that starts with "SECRET_" secrets="$(env | grep 'SECRET_')" || true -if [[ "$secrets" != "" ]]; then +if [[ "${secrets}" != "" ]]; then echo "Passing build secrets to sandbox..." - build-exec "echo '$secrets' | tee -a .docksal/docksal-local.env >/dev/null" + build-exec "echo '${secrets}' | tee -a .docksal/docksal-local.env >/dev/null" fi From 160f5f161a31bda822dd579dbac5a8b335c194da Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 14:05:18 -0700 Subject: [PATCH 09/16] Moved CI_SSH_KEY to optional vars in docs Since rsync is used by default to sync to codebase to the server, the CI_SSH_KEY is no longer a requirement since the sandbox server does not need access to the git repo to get the codebase. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 37efa9f..7593baf 100644 --- a/README.md +++ b/README.md @@ -59,15 +59,15 @@ If using `DOCKSAL_HOST_IP`, the agent will use `nip.io` for dynamic wildcard dom A base64 encoded private SSH key, used to access the remote Docksal host. -`CI_SSH_KEY` - -A base64 encoded private SSH key, used by default for all hosts (set as `Host *` in `~/.ssh/config`). -This key will be used to clone/push to repo, run commands over SSH on a remote deployment environment, etc. - Note: `cat /path/to/ | base64` can be used to create a base64 encoded string from a private SSH key. ### Optional +`CI_SSH_KEY` + +A base64 encoded private SSH key, used by default for all hosts (set as `Host *` in `~/.ssh/config`). +This key will be used to clone/push to git, run commands over SSH on a remote deployment environment, etc. + `DOCKSAL_DOMAIN` Can be used to set the base URL for sandbox builds (defaults to `DOCKSAL_HOST` if not set), individually from `DOCKSAL_HOST`. From d570a089a0f3101052a388c0881bc4eb8c6b6c55 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 16:47:29 -0700 Subject: [PATCH 10/16] Added ssh-agent initialization in build-env --- base/bin/build-env | 16 ++++++++++++++++ tests/base.bats | 13 ++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 6a0d68a..4636c00 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -162,17 +162,33 @@ build_env () # Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values. ssh_init () { + # Default key used for all hosts if [[ "$CI_SSH_KEY" != "" ]]; then umask 077 echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa chmod 0600 $HOME/.ssh/id_rsa fi + # Docksal Sandbox server key if [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]]; then umask 077 echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa chmod 0600 $HOME/.ssh/docksal_host_id_rsa fi + + # Initialize ssh-agent and load the default key ($HOME/.ssh/id_rsa) + # Check whether ssh-agent is configured + ssh-add -l &>/dev/null + # If ssh-agent is not configured, but config file exists, attempt to load agent settings from the file + [[ "$?" == 2 ]] && [[ -f ~/.ssh-agent ]] && eval "$(<~/.ssh-agent)" >/dev/null + # Check whether ssh-agent is configured again + ssh-add -l &>/dev/null + # If the existing config was invalid, start a new agent, write new config and load keys into the new ssh-agent + if [[ "$?" != 0 ]]; then + (umask 066; ssh-agent > ~/.ssh-agent) + eval "$(<~/.ssh-agent)" >/dev/null + ssh-add >/dev/null + fi } # Configure preferred git settings diff --git a/tests/base.bats b/tests/base.bats index 678a6e3..995af2e 100755 --- a/tests/base.bats +++ b/tests/base.bats @@ -64,19 +64,26 @@ teardown() { make clean } -@test "Check SSH keys" { +@test "Check SSH agent and keys" { [[ $SKIP == 1 ]] && skip ### Setup ### - make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash" -e CI_SSH_KEY="dGVzdC1zc2gta2V5Cg=="' + CI_SSH_KEY="LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUkxOElNMTZEMFVsS0U0VHVYeU1iQ3NEb3VHWU9TZC85SkJmSTcrenFCQ1JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1BSR2RQOUpSTmljbVQ1RjR1WFNEakV2TXFUczAxaVVOTXprTXAzUVdSM3hScWp5VFlYdAp0R1hRNE5BVWlxWEtlMnNaN0NZMmxqeGNrTUJCamU2OEhBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=" + make start -e ENV="-e GIT_USER_EMAIL='git@example.com' -e GIT_USER_NAME='Docksal CLI' -e GIT_REPO_URL='test-repo-url' -e GIT_BRANCH_NAME='test-branch-name' -e GIT_COMMIT_HASH='test-commit-hash' -e CI_SSH_KEY='${CI_SSH_KEY}'" + make exec COMMAND="build-env" ### Tests ### + # Check private SSH key - run make exec COMMAND="build-env" run make exec COMMAND='bash -lc "echo \$$CI_SSH_KEY | base64 -d | diff \$$HOME/.ssh/id_rsa -"' [[ "$status" == 0 ]] unset output + # Check ssh-agent + run make exec COMMAND='bash -lc "source build-env; ssh-add -l"' + [[ "$status" == 0 ]] + unset output + ### Cleanup ### make clean } From 3cae5ad9c6aa145d3668520e13e067cfe056a11d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 16:47:49 -0700 Subject: [PATCH 11/16] Replaced spaces with tabs in base.bats --- tests/base.bats | 74 ++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/base.bats b/tests/base.bats index 995af2e..2491478 100755 --- a/tests/base.bats +++ b/tests/base.bats @@ -41,49 +41,49 @@ teardown() { } @test "Git settings" { - [[ $SKIP == 1 ]] && skip - - ### Setup ### - make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash"' - - ### Tests ### - # Check git settings were applied - run make exec COMMAND="build-env" - run make exec COMMAND="git config --get --global user.email" - [[ "$status" == 0 ]] - echo "$output" | grep "git@example.com" - unset output - - run make exec COMMAND="build-env" - run make exec COMMAND="git config --get --global user.name" - [[ "$status" == 0 ]] - echo "$output" | grep "Docksal CLI" - unset output - - ### Cleanup ### - make clean + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start -e ENV='-e GIT_USER_EMAIL=git@example.com -e GIT_USER_NAME="Docksal CLI" -e GIT_REPO_URL="test-repo-url" -e GIT_BRANCH_NAME="test-branch-name" -e GIT_COMMIT_HASH="test-commit-hash"' + + ### Tests ### + # Check git settings were applied + run make exec COMMAND="build-env" + run make exec COMMAND="git config --get --global user.email" + [[ "$status" == 0 ]] + echo "$output" | grep "git@example.com" + unset output + + run make exec COMMAND="build-env" + run make exec COMMAND="git config --get --global user.name" + [[ "$status" == 0 ]] + echo "$output" | grep "Docksal CLI" + unset output + + ### Cleanup ### + make clean } @test "Check SSH agent and keys" { - [[ $SKIP == 1 ]] && skip + [[ $SKIP == 1 ]] && skip - ### Setup ### - CI_SSH_KEY="LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUkxOElNMTZEMFVsS0U0VHVYeU1iQ3NEb3VHWU9TZC85SkJmSTcrenFCQ1JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1BSR2RQOUpSTmljbVQ1RjR1WFNEakV2TXFUczAxaVVOTXprTXAzUVdSM3hScWp5VFlYdAp0R1hRNE5BVWlxWEtlMnNaN0NZMmxqeGNrTUJCamU2OEhBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=" - make start -e ENV="-e GIT_USER_EMAIL='git@example.com' -e GIT_USER_NAME='Docksal CLI' -e GIT_REPO_URL='test-repo-url' -e GIT_BRANCH_NAME='test-branch-name' -e GIT_COMMIT_HASH='test-commit-hash' -e CI_SSH_KEY='${CI_SSH_KEY}'" - make exec COMMAND="build-env" + ### Setup ### + CI_SSH_KEY="LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUkxOElNMTZEMFVsS0U0VHVYeU1iQ3NEb3VHWU9TZC85SkJmSTcrenFCQ1JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1BSR2RQOUpSTmljbVQ1RjR1WFNEakV2TXFUczAxaVVOTXprTXAzUVdSM3hScWp5VFlYdAp0R1hRNE5BVWlxWEtlMnNaN0NZMmxqeGNrTUJCamU2OEhBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=" + make start -e ENV="-e GIT_USER_EMAIL='git@example.com' -e GIT_USER_NAME='Docksal CLI' -e GIT_REPO_URL='test-repo-url' -e GIT_BRANCH_NAME='test-branch-name' -e GIT_COMMIT_HASH='test-commit-hash' -e CI_SSH_KEY='${CI_SSH_KEY}'" + make exec COMMAND="build-env" - ### Tests ### + ### Tests ### - # Check private SSH key - run make exec COMMAND='bash -lc "echo \$$CI_SSH_KEY | base64 -d | diff \$$HOME/.ssh/id_rsa -"' - [[ "$status" == 0 ]] - unset output + # Check private SSH key + run make exec COMMAND='bash -lc "echo \$$CI_SSH_KEY | base64 -d | diff \$$HOME/.ssh/id_rsa -"' + [[ "$status" == 0 ]] + unset output - # Check ssh-agent - run make exec COMMAND='bash -lc "source build-env; ssh-add -l"' - [[ "$status" == 0 ]] - unset output + # Check ssh-agent + run make exec COMMAND='bash -lc "source build-env; ssh-add -l"' + [[ "$status" == 0 ]] + unset output - ### Cleanup ### - make clean + ### Cleanup ### + make clean } From d425167b3e4f2d55a0dd16b4cef516d417dd6e4e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 18:05:40 -0700 Subject: [PATCH 12/16] Fixing ssh_init --- base/bin/build-env | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 4636c00..ea5ece4 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -162,33 +162,37 @@ build_env () # Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values. ssh_init () { + mkdir -p $HOME/.ssh + # Default key used for all hosts if [[ "$CI_SSH_KEY" != "" ]]; then - umask 077 echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa chmod 0600 $HOME/.ssh/id_rsa fi # Docksal Sandbox server key if [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]]; then - umask 077 echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa chmod 0600 $HOME/.ssh/docksal_host_id_rsa fi # Initialize ssh-agent and load the default key ($HOME/.ssh/id_rsa) # Check whether ssh-agent is configured - ssh-add -l &>/dev/null + ssh-add -l &>/dev/null || ret=$? # If ssh-agent is not configured, but config file exists, attempt to load agent settings from the file - [[ "$?" == 2 ]] && [[ -f ~/.ssh-agent ]] && eval "$(<~/.ssh-agent)" >/dev/null + if [[ "${ret}" == 2 ]] && [[ -f $HOME/.ssh/agent ]]; then + eval "$(<$HOME/.ssh/agent)" >/dev/null + fi # Check whether ssh-agent is configured again - ssh-add -l &>/dev/null - # If the existing config was invalid, start a new agent, write new config and load keys into the new ssh-agent - if [[ "$?" != 0 ]]; then - (umask 066; ssh-agent > ~/.ssh-agent) - eval "$(<~/.ssh-agent)" >/dev/null - ssh-add >/dev/null + ssh-add -l &>/dev/null || ret=$? + # If the existing config was invalid, start a new agent and write new config + if [[ "${ret}" == 2 ]]; then + ssh-agent > $HOME/.ssh/agent + chmod 0600 $HOME/.ssh/agent + eval "$(<$HOME/.ssh/agent)" >/dev/null fi + # Load default keys into the ssh-agent if available + ssh-add >/dev/null || true } # Configure preferred git settings From ea7165fc944bde480bbdddc47cac4371d184db36 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 2 Nov 2018 22:42:11 -0700 Subject: [PATCH 13/16] Pass CI_SSH_KEY to sandbox Note the key is passed as SECRET_SSH_PRIVATE_KEY, which docksal/cli reads, decodes and stores as ~/.ssh/id_rsa --- base/bin/build-init | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/bin/build-init b/base/bin/build-init index f220bfc..a911f6c 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -41,6 +41,13 @@ if [[ "${SANDBOX_PERMANENT}" != "" ]]; then build-exec "echo SANDBOX_PERMANENT=${SANDBOX_PERMANENT} | tee -a .docksal/docksal-local.env" fi +# Pass CI_SSH_KEY to sandbox +# Note the key is passed as SECRET_SSH_PRIVATE_KEY, which docksal/cli reads, decodes and stores as ~/.ssh/id_rsa +if [[ "${CI_SSH_KEY}" != "" ]]; then + echo "Passing CI_SSH_KEY to sandbox..." + build-exec "echo SECRET_SSH_PRIVATE_KEY=\"${CI_SSH_KEY}\" | tee -a .docksal/docksal-local.env >/dev/null" +fi + # Pass build secrets to sandbox # A "secret" is any environment variable that starts with "SECRET_" secrets="$(env | grep 'SECRET_')" || true From ba14a6a6709c8c82eabc41145faa58534cf9413b Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Sun, 4 Nov 2018 00:04:14 -0700 Subject: [PATCH 14/16] Entirely mute ssh-add in build-env ssh-add writes status output to stderr --- base/bin/build-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index ea5ece4..966861a 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -192,7 +192,7 @@ ssh_init () eval "$(<$HOME/.ssh/agent)" >/dev/null fi # Load default keys into the ssh-agent if available - ssh-add >/dev/null || true + ssh-add &>/dev/null || true } # Configure preferred git settings From 755a0a74dd80dcbb9c9739239f0c30aa29eb3a22 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Sun, 4 Nov 2018 00:05:02 -0700 Subject: [PATCH 15/16] Fixed grep for secrets variables in build-init --- base/bin/build-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-init b/base/bin/build-init index a911f6c..aaf20d3 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -50,7 +50,7 @@ fi # Pass build secrets to sandbox # A "secret" is any environment variable that starts with "SECRET_" -secrets="$(env | grep 'SECRET_')" || true +secrets="$(env | grep '^SECRET_')" || true if [[ "${secrets}" != "" ]]; then echo "Passing build secrets to sandbox..." build-exec "echo '${secrets}' | tee -a .docksal/docksal-local.env >/dev/null" From 4f0ed54229c144c9704789c681cb446fbfa8be3d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 5 Nov 2018 09:50:21 -0800 Subject: [PATCH 16/16] Disabled passing CI_SSH_KEY to sandbox This may be a security concern, if a single shared machine-user SSH key is used across multiple projects. TODO: Load the key into the docksal/ssh-agent service on the sandbox server instead. --- base/bin/build-init | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/base/bin/build-init b/base/bin/build-init index aaf20d3..9dde965 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -43,10 +43,12 @@ fi # Pass CI_SSH_KEY to sandbox # Note the key is passed as SECRET_SSH_PRIVATE_KEY, which docksal/cli reads, decodes and stores as ~/.ssh/id_rsa -if [[ "${CI_SSH_KEY}" != "" ]]; then - echo "Passing CI_SSH_KEY to sandbox..." - build-exec "echo SECRET_SSH_PRIVATE_KEY=\"${CI_SSH_KEY}\" | tee -a .docksal/docksal-local.env >/dev/null" -fi +# Disabled for now. This may be a security concern, if a single shared machine-user SSH key is used across multiple projects. +# TODO: Load the key into the docksal/ssh-agent service on the sandbox server instead. +#if [[ "${CI_SSH_KEY}" != "" ]]; then +# echo "Passing CI_SSH_KEY to sandbox..." +# build-exec "echo SECRET_SSH_PRIVATE_KEY=\"${CI_SSH_KEY}\" | tee -a .docksal/docksal-local.env >/dev/null" +#fi # Pass build secrets to sandbox # A "secret" is any environment variable that starts with "SECRET_"