Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update for KUBE-IPTABLES-HINT (and other 2022-ness) #3

Merged
merged 3 commits into from
Apr 12, 2022
Merged

update for KUBE-IPTABLES-HINT (and other 2022-ness) #3

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e56c74e
Drop support for iptables 1.8.3, update distro version notes
danwinship Mar 31, 2022
2a9e1a1
Default to nft over legacy in ambiguous situations
danwinship Mar 31, 2022
70d7897
Try to resolve nft/legacy based on KUBE-IPTABLES-HINT / KUBE-KUBELET-…
danwinship Mar 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 27 additions & 9 deletions iptables-wrapper-installer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,34 @@ cat > "${sbin}/iptables-wrapper" <<EOF

set -eu

# Detect whether the base system is using iptables-legacy or
# iptables-nft. This assumes that some non-containerized process (eg
# kubelet) has already created some iptables rules.

num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
# In kubernetes 1.17 and later, kubelet will have created at least
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
# "KUBE-KUBELET-CANARY"), so check that first, against
# iptables-nft, because we can check that more efficiently and
# it's more common these days.
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
mode=nft
else
# Check for kubernetes 1.17-or-later with iptables-legacy. We
# can't pass "-t mangle" to iptables-legacy-save because it would
# cause the kernel to create that table if it didn't already
# exist, which we don't want. So we have to grab all the rules
Comment on lines +111 to +114
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🙃

legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
mode=legacy
else
# With older kubernetes releases there may not be any _specific_
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would say it's safe to assume KUBE-MARK-DROP will always exist -- it was added in, AFAICT, v1.4.

That said, it probably makes sense to just fall back to "literally any rules exist at all".

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

KUBE-MARK-DROP is going away 🙂
kubernetes/enhancements#3179

# rules we can look for, but we assume that some non-containerized process
# (possibly kubelet) will have created _some_ iptables rules.
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
mode=nft
fi
fi
fi

EOF
Expand Down
14 changes: 10 additions & 4 deletions test/run-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,20 @@ if ! build "${tag}" ${build_arg}; then
FAIL "build failed unexpectedly"
fi

if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh legacy; then
FAIL "failed legacy test"
if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh legacy old; then
FAIL "failed legacy iptables / old rules test"
fi
if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh nft; then
if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh legacy new; then
FAIL "failed legacy iptables / new rules test"
fi
if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh nft old; then
if [[ "${nft_fail}" = 1 ]]; then
PASS "nft mode failed as expected"
fi
FAIL "failed nft test"
FAIL "failed nft iptables / old rules test"
fi
if ! docker run --privileged "iptables-wrapper-test-${tag}" /bin/sh ${dash_x:-} /test.sh nft new; then
FAIL "failed nft iptables / new rules test"
fi

PASS "success"
Expand Down
60 changes: 47 additions & 13 deletions test/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,20 @@
set -eu

mode=$1
rulestype=$2

case "${mode}" in
legacy)
wrongmode=nft
;;
nft)
wrongmode=legacy
;;
*)
echo "ERROR: bad mode '${mode}'" 1>&2
exit 1
;;
esac

if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
sbin="/usr/sbin"
Expand Down Expand Up @@ -56,22 +70,42 @@ ensure_iptables_resolved() {

ensure_iptables_undecided

# Initialize the chosen iptables mode with a subset of kubelet's rules
iptables-${mode} -t nat -N KUBE-MARK-DROP
iptables-${mode} -t nat -A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000
iptables-${mode} -t filter -N KUBE-FIREWALL
iptables-${mode} -t filter -A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
iptables-${mode} -t filter -I OUTPUT -j KUBE-FIREWALL
iptables-${mode} -t filter -I INPUT -j KUBE-FIREWALL
case "${rulestype}" in
old)
# Initialize the chosen iptables mode with some kubelet-like rules
iptables-${mode} -t nat -N KUBE-MARK-DROP
iptables-${mode} -t nat -A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000
iptables-${mode} -t filter -N KUBE-FIREWALL
iptables-${mode} -t filter -A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
iptables-${mode} -t filter -I OUTPUT -j KUBE-FIREWALL
iptables-${mode} -t filter -I INPUT -j KUBE-FIREWALL

# Fail on iptables 1.8.2 in nft mode
if ! iptables-${mode} -C KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP; then
echo "failed to match previously-added rule; iptables is broken" 1>&2
exit 1
fi
;;

new)
# Initialize the chosen iptables mode with just a hint chain
iptables-${mode} -t mangle -N KUBE-IPTABLES-HINT
;;

*)
echo "ERROR: bad rulestype '${rulestype}'" 1>&2
exit 1
;;
esac

# Put some junk in the other iptables system
iptables-${wrongmode} -t filter -N BAD-1
iptables-${wrongmode} -t filter -A BAD-1 -j ACCEPT
iptables-${wrongmode} -t filter -N BAD-2
iptables-${wrongmode} -t filter -A BAD-2 -j DROP

ensure_iptables_undecided

iptables -L > /dev/null

ensure_iptables_resolved ${mode}

# Fail on iptables 1.8.2 in nft mode
if ! iptables -C KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP; then
echo "failed to match previously-added rule; iptables is broken" 1>&2
exit 1
fi