Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into patch-1
Browse files Browse the repository at this point in the history
  • Loading branch information
lhotari committed Dec 11, 2023
2 parents e1a9b94 + a8d667a commit 7ff49f1
Show file tree
Hide file tree
Showing 46 changed files with 656 additions and 959 deletions.
17 changes: 17 additions & 0 deletions .ci/clusters/values-broker-tls.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,15 @@ components:

zookeeper:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false

bookkeeper:
replicaCount: 3
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
configData:
diskUsageThreshold: "0.999"
diskUsageWarnThreshold: "0.999"
Expand All @@ -50,6 +56,9 @@ bookkeeper:

broker:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false
configData:
## Enable `autoSkipNonRecoverableData` since bookkeeper is running
## without persistence
Expand All @@ -59,8 +68,16 @@ broker:
managedLedgerDefaultWriteQuorum: "1"
managedLedgerDefaultAckQuorum: "1"

autorecovery:
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false

proxy:
replicaCount: 1
# Disable pod monitor since we're disabling CRD installation
podMonitor:
enabled: false

toolset:
useProxy: false
Expand Down
161 changes: 161 additions & 0 deletions .github/actions/ssh-access/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: ssh access
description: Sets up SSH access to build VM with upterm
inputs:
action:
description: |
Action to perform: options are "start" and "wait"
"start" will install, configure and start upterm.
"wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
required: false
default: 'start'
limit-access-to-actor:
description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
required: false
default: 'false'
limit-access-to-users:
description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
required: false
default: ''
secure-access:
description: |
Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
required: false
default: 'true'
timeout:
description: 'When action=wait, the timeout in seconds to wait for the user to connect'
required: false
default: '300'
runs:
using: composite
steps:
- run: |
if [[ "${{ inputs.action }}" == "start" ]]; then
echo "::group::Installing upterm & tmux"
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
# install upterm
curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
sudo apt-get -y install tmux
fi
elif [[ "$OSTYPE" == "darwin"* ]]; then
brew install owenthereal/upterm/upterm
# install tmux if it's not present
if ! command -v tmux &>/dev/null; then
brew install tmux
fi
else
echo "Unsupported $OSTYPE"
exit 0
fi
echo '::endgroup::'
echo "::group::Configuring ssh and ssh keys"
# generate ssh key
mkdir -p ~/.ssh
chmod 0700 ~/.ssh
if [ ! -f ~/.ssh/id_rsa ]; then
ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
fi
if [ ! -f ~/.ssh/id_ed25519 ]; then
ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
fi
# configure ssh
echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
# Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
# @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
authorizedKeysParameter=""
authorizedKeysFile=${HOME}/.ssh/authorized_keys
if [[ "${{ inputs.secure-access }}" != "false" ]]; then
ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
cat /tmp/dummykey$$.pub >> $authorizedKeysFile
rm /tmp/dummykey$$ /tmp/dummykey$$.pub
fi
limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
if [[ "${limit_access_to_actor}" == "true" ]]; then
echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
fi
limit_access_to_users="${{ inputs.limit-access-to-users }}"
for github_user in ${limit_access_to_users//,/ }; do
if [[ -n "${github_user}" ]]; then
echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
fi
done
if [ -f $authorizedKeysFile ]; then
chmod 0600 $authorizedKeysFile
authorizedKeysParameter="-a $authorizedKeysFile"
echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
cat $authorizedKeysFile
echo "---------------------------"
fi
echo '::endgroup::'
echo "::group::Starting terminal session and connecting to server"
tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
sleep 2
tmux send-keys -t upterm-wrapper q C-m
sleep 1
tmux set -t upterm-wrapper window-size largest
tmux set -t upterm window-size largest
echo '::endgroup::'
echo -e "\nSSH connection information"
# wait up to 10 seconds for upterm admin socket to appear
for i in {1..10}; do
ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
if [ ! -S "$ADMIN_SOCKET" ]; then
echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
sleep 1
else
echo "upterm admin socket available in $ADMIN_SOCKET"
break
fi
done
shopt -s nullglob
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "Starting upterm failed."
exit 0
}
elif [[ "${{ inputs.action }}" == "wait" ]]; then
# only wait if upterm was installed
if command -v upterm &>/dev/null; then
shopt -s nullglob
echo "SSH connection information"
upterm session current --admin-socket ~/.upterm/*.sock || {
echo "upterm isn't running. Not waiting any longer."
exit 0
}
timeout=${{ inputs.timeout }}
echo "Waiting $timeout seconds..."
sleep $timeout
echo "Keep waiting as long as there's a connected session"
while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
sleep 30
done
echo "No session is connected. Not waiting any longer."
else
echo "upterm isn't installed"
fi
fi
shell: bash
40 changes: 30 additions & 10 deletions .github/actions/tune-runner-vm/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ runs:
steps:
- run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled properly
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
Expand All @@ -32,18 +33,23 @@ runs:
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
# Set swappiness to 1 for all cgroups and sub-groups
for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do
if [ -d "swappiness_dir" ]; then
echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null
fi
done
(
shopt -s nullglob
# Set swappiness to 1 for all cgroups and sub-groups
for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
echo 1 | sudo tee $swappiness_file > /dev/null
done
) || true
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
# Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
Expand All @@ -70,13 +76,27 @@ runs:
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
# enable docker experimental mode which is
# required for using "docker build --squash" / "-Ddocker.squash=true"
daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')"
echo "$daemon_json" | sudo tee /etc/docker/daemon.json
# restart docker daemon
sudo systemctl restart docker
echo '::endgroup::'
# show memory
echo "::group::Available Memory"
free -m
echo '::endgroup::'
# show disk
df -h
echo "::group::Available diskspace"
df -BM
echo "::endgroup::"
# show cggroup
echo "/actions_job cgroup settings:"
sudo cgget actions_job
echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
sudo cgget -a $CURRENT_CGGROUP || true
echo '::endgroup::'
fi
shell: bash
106 changes: 0 additions & 106 deletions .github/workflows/cancel-duplicate-workflows.yml

This file was deleted.

Loading

0 comments on commit 7ff49f1

Please sign in to comment.