-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit updates etcd defrag and backup script as follows: - Script exits without any defrag/backup/trim action if: - It is executed on non leader etcd member - It is executed on single member etcd cluster - Script defragment the etcd cluster as follows: - Defrag the non leader etcd members first - Change the leadership to the randomly selected and defragmentation completed etcd member - Defrag the local (ex-leader) etcd member - Script then backup & trim local (ex-leader) etcd member This script executes etcdctl commands like `etcdctl move-leader` or `etcdctl endpoint status --cluster` which were introduced in etcdctl version 3.3.0. The previous etcdctl client was installed as an `apt` package. The latest etcdctl version available in Ubuntu 20.04 repositories is v3.2.26, hence this commit also introduces `etcdctl_version` variable that contains the desired version of etcdctl client. Etcdctl client is then used for etcd DB maintenance tasks. Issue #384 Signed-off-by: Matej Feder <matej.feder@dnation.cloud>
- Loading branch information
Showing
9 changed files
with
89 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
#!/bin/bash | ||
# Defragment & backup & trim script for SCS k8s-cluster-api-provider etcd cluster | ||
# Script exits without any defrag/backup/trim action if: | ||
# - It is executed on non leader etcd member | ||
# - It is executed on single member etcd cluster | ||
# Script defragment the etcd cluster as follows: | ||
# - Defrag the non leader etcd members first | ||
# - Change the leadership to the randomly selected and defragmentation completed etcd member | ||
# - Defrag the local (ex-leader) etcd member | ||
# Script then backup & trim local (ex-leader) etcd member | ||
# | ||
# Usage: etcd-defrag.sh | ||
|
||
export LOG_DIR=/var/log | ||
export ETCDCTL_API=3 | ||
ETCDCTL="etcdctl --cert /etc/kubernetes/pki/etcd/peer.crt --key /etc/kubernetes/pki/etcd/peer.key --cacert /etc/kubernetes/pki/etcd/ca.crt" | ||
|
||
if test "$($ETCDCTL endpoint status | cut -d ',' -f 5 | tr -d [:blank:])" != "false"; then | ||
echo "Exit on non leader" | ||
exit 0 | ||
fi | ||
|
||
# Get all etcd members with their endpoints, IDs, and leader status | ||
declare -a MEMBERS | ||
while read MEMBER; do | ||
MEMBERS+=( "$MEMBER" ) | ||
done < <($ETCDCTL endpoint status --cluster) | ||
|
||
if test ${#MEMBERS[@]} = 1; then | ||
echo "Exit on single member etcd" | ||
exit 0 | ||
fi | ||
|
||
declare -a NON_LEADER_IDS | ||
for MEMBER in "${MEMBERS[@]}"; do | ||
# Get member ID, endpoint, and leader status | ||
MEMBER_ENDPOINT=$(echo "$MEMBER" | cut -d ',' -f 1 | tr -d [:blank:]) | ||
MEMBER_ID=$(echo "$MEMBER" | cut -d ',' -f 2 | tr -d [:blank:]) | ||
MEMBER_IS_LEADER=$(echo "$MEMBER" | cut -d ',' -f 5 | tr -d [:blank:]) | ||
# Defragment if $MEMBER is not the leader | ||
if test "$MEMBER_IS_LEADER" == "false"; then | ||
echo "Etcd member ${MEMBER_ENDPOINT} is not the leader, let's defrag it!" | ||
$ETCDCTL --endpoints="$MEMBER_ENDPOINT" defrag | ||
NON_LEADER_IDS+=( "$MEMBER_ID" ) | ||
fi | ||
done | ||
|
||
# Randomly pick an ID from non-leader IDs and make it a leader | ||
RANDOM_NON_LEADER_ID=${NON_LEADER_IDS[ $(($RANDOM % ${#NON_LEADER_IDS[@]})) ]} | ||
echo "Member ${RANDOM_NON_LEADER_ID} is becoming the leader" | ||
$ETCDCTL move-leader $RANDOM_NON_LEADER_ID | ||
|
||
# Defrag this ex-leader etcd member | ||
sync | ||
sleep 2 | ||
$ETCDCTL defrag | ||
|
||
# Backup&trim this ex-leader etcd member | ||
sleep 3 | ||
$ETCDCTL snapshot save /root/etcd-backup | ||
chmod 0600 /root/etcd-backup | ||
xz -f /root/etcd-backup | ||
fstrim -v /var/lib/etcd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters