Skip to content

Commit

Permalink
[autoscaler] fix conflits
Browse files Browse the repository at this point in the history
  • Loading branch information
zyjiaobj committed Jul 22, 2019
2 parents 812e89e + 2322308 commit b7edbff
Show file tree
Hide file tree
Showing 141 changed files with 4,356 additions and 1,093 deletions.
2 changes: 2 additions & 0 deletions deploy/Dockerfile.all-in-one
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ COPY --from=builder /home/stratos/dev-certs /srv/dev-certs
COPY --from=builder /home/stratos/ui /srv/ui
COPY --from=builder /home/stratos/jetstream /srv/jetstream
COPY --from=builder /home/stratos/config.properties /srv/config.properties
# User Invite templates
COPY --from=builder /home/stratos/src/jetstream/templates /srv/templates

EXPOSE 443

Expand Down
2 changes: 2 additions & 0 deletions deploy/all-in-one/config.all-in-one.properties
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,5 @@ SESSION_STORE_SECRET=wheeee!
ENCRYPTION_KEY=B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF
STRATOS_DEPLOYMENT_DOCKER_AIO=true
SKIP_SSL_VALIDATION=true

TEMPLATE_DIR=./templates
10 changes: 10 additions & 0 deletions deploy/ci/automation/cfallinonetest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,18 @@ CONTAINER_ID=$(docker run \
-e UAA_ENDPOINT="${UAA}" \
-e SKIP_SSL_VALIDATION='true' \
-e CONSOLE_ADMIN_SCOPE='cloud_controller.admin' \
-e SMTP_HOST="${SMTP_HOST}" \
-e SMTP_FROM_ADDRESS="${SMTP_FROM_ADDRESS}" \
$IMAGE)

# Show backend log - wait a few seconds for it to start up

echo "Backend startup log ..."
sleep 20
docker logs ${CONTAINER_ID} | tail -n 100

echo "Preparing E2E Tests..."

# Need node modules to run the tests
rm -rf node_modules
npm install
Expand Down
31 changes: 29 additions & 2 deletions deploy/ci/automation/cfpushtest.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
#!/bin/bash

GREEN='\033[0;32m'
RED='\033[0;31m'
CYAN="\033[96m"
YELLOW="\033[93m"
BOLD="\033[1m"
RESET='\033[0m'

echo "===================="
echo "Stratos CF Push Test"
echo "===================="
Expand Down Expand Up @@ -83,6 +90,15 @@ echo " env:" >> $MANIFEST
echo " SKIP_AUTO_REGISTER: true" >> $MANIFEST
echo " FORCE_ENABLE_PERSISTENCE_FEATURES: true" >> $MANIFEST

# Make sure we add invite users config if set
if [ -n "${SMTP_HOST}" ]; then
echo " SMTP_HOST: ${SMTP_HOST}" >> $MANIFEST
fi

if [ -n "${SMTP_FROM_ADDRESS}" ]; then
echo " SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS}" >> $MANIFEST
fi

# SSO
SUITE=""
if [ "$2" == "sso" ] || [ "$3" == "sso" ] ; then
Expand All @@ -101,7 +117,7 @@ cat $MANIFEST

# Prebuild
if [ "$2" == "prebuild" ]; then
echo "Pre-building UI ..."
echo -e "${CYAN}Pre-building UI ...${RESET}"
npm install
npm run prebuild-ui
fi
Expand All @@ -118,9 +134,20 @@ date
if [ $RET -ne 0 ]; then
set +e
echo "Push failed... showing recent log of the Stratos app"
cf logs console --recent
cf logs --recent console
set -e
else

# Show the recent logs just we can see startup settings
echo -e "${BOLD}${GREEN}Showing recent logs of the Stratos App${RESET}"
cf logs --recent console | tail -n 100

echo -e "${BOLD}${GREEN}"
echo "==============================================================================="
echo ""
echo "Running E2E Tests...."
echo -e "${RESET}"

# Push was okay, so we can prepare and run E2E tests
rm -rf node_modules
npm install
Expand Down
4 changes: 4 additions & 0 deletions deploy/ci/automation/e2e-clean-remnants.sh
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,8 @@ echo "Cleaning test Users"
USERS=$(cf space-users e2e e2e | grep "accept" | sed -e 's/^[[:space:]]*//')
clean "$USERS" "-" "delete-user" "^(acceptancee2etravis)(invite[0-9])(20[0-9]*)[Tt]([0-9]*)[zZ].*"

# user -a with org users so we get all users (including those without roles)
USERS=$(cf org-users -a e2e | grep "accept" | sed -e 's/^[[:space:]]*//')
clean "$USERS" "-" "delete-user" "^(acceptancee2etravis)(invite[0-9])(20[0-9]*)[Tt]([0-9]*)[zZ].*"

echo "Done"
6 changes: 6 additions & 0 deletions deploy/ci/automation/helmtest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ function deleteRelease {
function waitForHelmRelease {
echo "Waiting for Stratos Helm Release to be ready..."
local DONE="false"
local TIMEOUT=0
while [ $DONE != "true" ]; do
COUNT=$(kubectl get po --namespace=${NAMESPACE} | wc -l)
kubectl get po --namespace=${NAMESPACE}
Expand All @@ -63,6 +64,11 @@ function waitForHelmRelease {
if [ "$DONE" != "true" ]; then
echo "Waiting for Stratos Helm release to be ready..."
sleep 5
TIMEOUT=$((TIMEOUT+1))
if [ ${TIMEOUT} -gt 60 ]; then
echo "Timed out waiting for Helm release to be ready"
exit 1
fi
fi
done
}
Expand Down
3 changes: 2 additions & 1 deletion deploy/ci/scripts/build-docker-image-resource.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ mkdir -p ./tmp
cd ./tmp
git clone https://github.com/concourse/docker-image-resource.git
cp ../docker-image-out-asset ./docker-image-resource/assets/out
cp ../docker-image-common.sh ./docker-image-resource/assets/common.sh
chmod +x ./docker-image-resource/assets/out

docker build ./docker-image-resource -t splatform/stratos-concourse-docker-image-resource:latest
docker build ./docker-image-resource -f ./docker-image-resource/dockerfiles/alpine/Dockerfile -t splatform/stratos-concourse-docker-image-resource:latest
docker push splatform/stratos-concourse-docker-image-resource:latest
rm -rf ./tmp
echo "All done"
210 changes: 210 additions & 0 deletions deploy/ci/scripts/docker-image-common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
LOG_FILE=${LOG_FILE:-/tmp/docker.log}
SKIP_PRIVILEGED=${SKIP_PRIVILEGED:-false}
STARTUP_TIMEOUT=${STARTUP_TIMEOUT:-120}

sanitize_cgroups() {
mkdir -p /sys/fs/cgroup
mountpoint -q /sys/fs/cgroup || \
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup

mount -o remount,rw /sys/fs/cgroup

sed -e 1d /proc/cgroups | while read sys hierarchy num enabled; do
if [ "$enabled" != "1" ]; then
# subsystem disabled; skip
continue
fi

grouping="$(cat /proc/self/cgroup | cut -d: -f2 | grep "\\<$sys\\>")" || true
if [ -z "$grouping" ]; then
# subsystem not mounted anywhere; mount it on its own
grouping="$sys"
fi

mountpoint="/sys/fs/cgroup/$grouping"

mkdir -p "$mountpoint"

# clear out existing mount to make sure new one is read-write
if mountpoint -q "$mountpoint"; then
umount "$mountpoint"
fi

mount -n -t cgroup -o "$grouping" cgroup "$mountpoint"

if [ "$grouping" != "$sys" ]; then
if [ -L "/sys/fs/cgroup/$sys" ]; then
rm "/sys/fs/cgroup/$sys"
fi

ln -s "$mountpoint" "/sys/fs/cgroup/$sys"
fi
done

if ! test -e /sys/fs/cgroup/systemd ; then
mkdir /sys/fs/cgroup/systemd
mount -t cgroup -o none,name=systemd none /sys/fs/cgroup/systemd
fi
}

start_docker() {
mkdir -p /var/log
mkdir -p /var/run

if [ "$SKIP_PRIVILEGED" = "false" ]; then
sanitize_cgroups

# check for /proc/sys being mounted readonly, as systemd does
if grep '/proc/sys\s\+\w\+\s\+ro,' /proc/mounts >/dev/null; then
mount -o remount,rw /proc/sys
fi
fi

local mtu=$(cat /sys/class/net/$(ip route get 8.8.8.8|awk '{ print $5 }')/mtu)
local server_args="--experimental --mtu ${mtu}"
local registry=""

server_args="${server_args} --max-concurrent-downloads=$1 --max-concurrent-uploads=$2"

for registry in $3; do
server_args="${server_args} --insecure-registry ${registry}"
done

if [ -n "$4" ]; then
server_args="${server_args} --registry-mirror $4"
fi

try_start() {
dockerd --data-root /scratch/docker ${server_args} >$LOG_FILE 2>&1 &
echo $! > /tmp/docker.pid

sleep 1

echo waiting for docker to come up...
until docker info >/dev/null 2>&1; do
sleep 1
if ! kill -0 "$(cat /tmp/docker.pid)" 2>/dev/null; then
return 1
fi
done
}

export server_args LOG_FILE
declare -fx try_start
trap stop_docker EXIT

if ! timeout ${STARTUP_TIMEOUT} bash -ce 'while true; do try_start && break; done'; then
echo Docker failed to start within ${STARTUP_TIMEOUT} seconds.
return 1
fi
}

stop_docker() {
local pid=$(cat /tmp/docker.pid)
if [ -z "$pid" ]; then
return 0
fi

kill -TERM $pid
}

log_in() {
local username="$1"
local password="$2"
local registry="$3"

if [ -n "${username}" ] && [ -n "${password}" ]; then
echo "${password}" | docker login -u "${username}" --password-stdin ${registry}
else
mkdir -p ~/.docker
echo '{"credsStore":"ecr-login"}' >> ~/.docker/config.json
fi
}

private_registry() {
local repository="${1}"

if echo "${repository}" | fgrep -q '/' ; then
local registry="$(extract_registry "${repository}")"
if echo "${registry}" | fgrep -q '.' ; then
return 0
fi
fi

return 1
}

extract_registry() {
local repository="${1}"

echo "${repository}" | cut -d/ -f1
}

extract_repository() {
local long_repository="${1}"

echo "${long_repository}" | cut -d/ -f2-
}

image_from_tag() {
docker images --no-trunc "$1" | awk "{if (\$2 == \"$2\") print \$3}"
}

image_from_digest() {
docker images --no-trunc --digests "$1" | awk "{if (\$3 == \"$2\") print \$4}"
}

certs_to_file() {
local raw_ca_certs="${1}"
local cert_count="$(echo $raw_ca_certs | jq -r '. | length')"

for i in $(seq 0 $(expr "$cert_count" - 1));
do
local cert_dir="/etc/docker/certs.d/$(echo $raw_ca_certs | jq -r .[$i].domain)"
mkdir -p "$cert_dir"
echo $raw_ca_certs | jq -r .[$i].cert >> "${cert_dir}/ca.crt"
done
}

set_client_certs() {
local raw_client_certs="${1}"
local cert_count="$(echo $raw_client_certs | jq -r '. | length')"

for i in $(seq 0 $(expr "$cert_count" - 1));
do
local cert_dir="/etc/docker/certs.d/$(echo $raw_client_certs | jq -r .[$i].domain)"
[ -d "$cert_dir" ] || mkdir -p "$cert_dir"
echo $raw_client_certs | jq -r .[$i].cert >> "${cert_dir}/client.cert"
echo $raw_client_certs | jq -r .[$i].key >> "${cert_dir}/client.key"
done
}

docker_pull() {
GREEN='\033[0;32m'
RED='\033[0;31m'
NC='\033[0m' # No Color

pull_attempt=1
max_attempts=3
while [ "$pull_attempt" -le "$max_attempts" ]; do
printf "Pulling ${GREEN}%s${NC}" "$1"

if [ "$pull_attempt" != "1" ]; then
printf " (attempt %s of %s)" "$pull_attempt" "$max_attempts"
fi

printf "...\n"

if docker pull "$1"; then
printf "\nSuccessfully pulled ${GREEN}%s${NC}.\n\n" "$1"
return 0
fi

echo

pull_attempt=$(expr "$pull_attempt" + 1)
done

printf "\n${RED}Failed to pull image %s.${NC}" "$1"
return 1
}
Loading

0 comments on commit b7edbff

Please sign in to comment.