Skip to content

Commit

Permalink
Merge pull request #510 from wongma7/ebshack
Browse files Browse the repository at this point in the history
Fast-forward to latest ebs hack/e2e scripts with eksctl support, k8s 1.20, etc.
  • Loading branch information
k8s-ci-robot authored Jul 14, 2021
2 parents f8f6b54 + b980f03 commit 2b75682
Show file tree
Hide file tree
Showing 8 changed files with 387 additions and 80 deletions.
11 changes: 10 additions & 1 deletion hack/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,16 @@ To commit changes and submit them as a PR back to the ebs repo:

```
git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff
cd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver
pushd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver
git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
git commit
```

To consume newer changes from the ebs repo:

```
git fetch ebs
git diff HEAD:hack/e2e ebs/master:hack/e2e > /tmp/hack_e2e.diff
git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
git commit
```
6 changes: 4 additions & 2 deletions hack/e2e/ebs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@ BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function ebs_check_migration() {
KUBECONFIG=${1}

loudecho "Checking migration"
# There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
# Find the controller-manager log and read its metrics to verify
NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &
NODE=$(kubectl get node -l kubernetes.io/role=master -o json --kubeconfig "${KUBECONFIG}" | jq -r ".items[].metadata.name")
kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system --kubeconfig "${KUBECONFIG}" &

# Ensure port forwarding succeeded
n=0
Expand Down
2 changes: 1 addition & 1 deletion hack/e2e/ecr.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ function ecr_build_and_push() {
IMAGE_NAME=${3}
IMAGE_TAG=${4}
set +e
if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then
set -e
loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
else
Expand Down
122 changes: 122 additions & 0 deletions hack/e2e/eksctl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/bin/bash

set -euo pipefail

function eksctl_install() {
INSTALL_PATH=${1}
EKSCTL_VERSION=${2}
if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
chmod +x "${INSTALL_PATH}"/eksctl
fi
}

function eksctl_create_cluster() {
SSH_KEY_PATH=${1}
CLUSTER_NAME=${2}
BIN=${3}
ZONES=${4}
INSTANCE_TYPE=${5}
K8S_VERSION=${6}
CLUSTER_FILE=${7}
KUBECONFIG=${8}
EKSCTL_PATCH_FILE=${9}
EKSCTL_ADMIN_ROLE=${10}
WINDOWS=${11}

generate_ssh_key "${SSH_KEY_PATH}"

CLUSTER_NAME="${CLUSTER_NAME//./-}"

if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then
loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
${BIN} upgrade cluster -f "${CLUSTER_FILE}"
else
loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
${BIN} create cluster \
--managed \
--ssh-access \
--ssh-public-key "${SSH_KEY_PATH}".pub \
--zones "${ZONES}" \
--nodes=3 \
--instance-types="${INSTANCE_TYPE}" \
--version="${K8S_VERSION}" \
--disable-pod-imds \
--dry-run \
"${CLUSTER_NAME}" > "${CLUSTER_FILE}"

if test -f "$EKSCTL_PATCH_FILE"; then
eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
fi

loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
fi

loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"

loudecho "Getting cluster ${CLUSTER_NAME}"
${BIN} get cluster "${CLUSTER_NAME}"

if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then
AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
fi

if [[ "$WINDOWS" == true ]]; then
${BIN} create nodegroup \
--cluster="${CLUSTER_NAME}" \
--node-ami-family=WindowsServer2019FullContainer \
-n ng-windows \
-m 1 \
-M 1 \
--ssh-access \
--ssh-public-key "${SSH_KEY_PATH}".pub
${BIN} utils install-vpc-controllers --cluster="${CLUSTER_NAME}" --approve
fi

return $?
}

function eksctl_cluster_exists() {
CLUSTER_NAME=${1}
BIN=${2}
set +e
if ${BIN} get cluster "${CLUSTER_NAME}"; then
set -e
return 0
else
set -e
return 1
fi
}

function eksctl_delete_cluster() {
BIN=${1}
CLUSTER_NAME=${2}
loudecho "Deleting cluster ${CLUSTER_NAME}"
${BIN} delete cluster "${CLUSTER_NAME}"
}

function eksctl_patch_cluster_file() {
CLUSTER_FILE=${1} # input must be yaml
EKSCTL_PATCH_FILE=${2} # input must be yaml

loudecho "Patching cluster $CLUSTER_NAME with $EKSCTL_PATCH_FILE"

# Temporary intermediate files for patching
CLUSTER_FILE_0=$CLUSTER_FILE.0
CLUSTER_FILE_1=$CLUSTER_FILE.1

cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"

# Patch only the Cluster
kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1"
mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"

# Done patching, overwrite original CLUSTER_FILE
mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
}
148 changes: 116 additions & 32 deletions hack/e2e/kops.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

set -uo pipefail
set -euo pipefail

OS_ARCH=$(go env GOOS)-amd64

Expand All @@ -19,53 +19,137 @@ function kops_install() {

function kops_create_cluster() {
SSH_KEY_PATH=${1}
KOPS_STATE_FILE=${2}
CLUSTER_NAME=${3}
KOPS_BIN=${4}
ZONES=${5}
CLUSTER_NAME=${2}
BIN=${3}
ZONES=${4}
NODE_COUNT=${5}
INSTANCE_TYPE=${6}
K8S_VERSION=${7}
TEST_DIR=${8}
KOPS_FEATURE_GATES_FILE=${10}
KOPS_ADDITIONAL_POLICIES_FILE=${11}
CLUSTER_FILE=${8}
KUBECONFIG=${9}
KOPS_PATCH_FILE=${10}
KOPS_PATCH_NODE_FILE=${11}
KOPS_STATE_FILE=${12}

loudecho "Generating SSH key $SSH_KEY_PATH"
if [[ ! -e ${SSH_KEY_PATH} ]]; then
ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
fi
generate_ssh_key "${SSH_KEY_PATH}"

set +e
if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
set -e
loudecho "Updating cluster $CLUSTER_NAME"
if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
else
set -e
loudecho "Creating cluster $CLUSTER_NAME"
${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
${BIN} create cluster --state "${KOPS_STATE_FILE}" \
--ssh-public-key="${SSH_KEY_PATH}".pub \
--zones "${ZONES}" \
--node-count=3 \
--node-count="${NODE_COUNT}" \
--node-size="${INSTANCE_TYPE}" \
--kubernetes-version="${K8S_VERSION}" \
--ssh-public-key="${SSH_KEY_PATH}".pub \
"${CLUSTER_NAME}"
--dry-run \
-o yaml \
"${CLUSTER_NAME}" > "${CLUSTER_FILE}"

if test -f "$KOPS_PATCH_FILE"; then
kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
fi
if test -f "$KOPS_PATCH_NODE_FILE"; then
kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
fi

loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
kops create secret --state "${KOPS_STATE_FILE}" --name "${CLUSTER_NAME}" sshpublickey admin -i "${SSH_KEY_PATH}".pub
fi

CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
[ -r "$KOPS_FEATURE_GATES_FILE" ] && cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
[ -r "$KOPS_ADDITIONAL_POLICIES_FILE" ] && cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes

loudecho "Validating cluster $CLUSTER_NAME"
${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}"
${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"

loudecho "Validating cluster ${CLUSTER_NAME}"
${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
return $?
}

function kops_cluster_exists() {
CLUSTER_NAME=${1}
BIN=${2}
KOPS_STATE_FILE=${3}
set +e
if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
set -e
return 0
else
set -e
return 1
fi
}

function kops_delete_cluster() {
KOPS_BIN=${1}
BIN=${1}
CLUSTER_NAME=${2}
KOPS_STATE_FILE=${3}
loudecho "Deleting cluster ${CLUSTER_NAME}"
${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
}

# TODO switch this to python, work exclusively with yaml, use kops toolbox
# template/kops set?, all this hacking with jq stinks!
function kops_patch_cluster_file() {
CLUSTER_FILE=${1} # input must be yaml
KOPS_PATCH_FILE=${2} # input must be yaml
KIND=${3} # must be either Cluster or InstanceGroup
ROLE=${4} # must be either Master or Node

loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"

# Temporary intermediate files for patching, don't mutate CLUSTER_FILE until
# the end
CLUSTER_FILE_JSON=$CLUSTER_FILE.json
CLUSTER_FILE_0=$CLUSTER_FILE.0
CLUSTER_FILE_1=$CLUSTER_FILE.1

# HACK convert the multiple yaml documents to an array of json objects
yaml_to_json "$CLUSTER_FILE" "$CLUSTER_FILE_JSON"

# Find the json objects to patch
FILTER=".[] | select(.kind==\"$KIND\")"
if [ -n "$ROLE" ]; then
FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
fi
jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"

# Patch only the json objects
kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"

# Delete the original json objects, add the patched
# TODO Cluster must always be first?
jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"

# HACK convert the array of json objects to multiple yaml documents
json_to_yaml "$CLUSTER_FILE_0" "$CLUSTER_FILE_1"
mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"

# Done patching, overwrite original yaml CLUSTER_FILE
mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml

# Clean up
rm "$CLUSTER_FILE_JSON"
}

function yaml_to_json() {
IN=${1}
OUT=${2}
kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
}

function json_to_yaml() {
IN=${1}
OUT=${2}
for ((i = 0; i < $(jq length "$IN"); i++)); do
echo "---" >> "$OUT"
jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
done
}
Loading

0 comments on commit 2b75682

Please sign in to comment.