diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index cd408aec78e..097fcc3e8dd 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -28,104 +28,9 @@ env: SUBMARINER_VERSION: '0.16.3' jobs: - build-kube-ovn-base: - name: Build kube-ovn-base - runs-on: ubuntu-22.04 - outputs: - build-base: ${{ steps.check.outputs.build-base }} - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - id: check - run: | - if [ ${{ github.event_name }} != 'pull_request' ]; then - exit - fi - if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base$; then - echo build-base=1 >> "$GITHUB_OUTPUT" - fi - - - uses: docker/setup-buildx-action@v3 - if: steps.check.outputs.build-base == 1 - - - name: Build kube-ovn-base image - id: build - if: steps.check.outputs.build-base == 1 - run: | - make base-amd64 - make base-tar-amd64 - - - name: Upload base images to artifact - if: steps.check.outputs.build-base == 1 - uses: actions/upload-artifact@v4 - with: - name: kube-ovn-base - path: image-amd64.tar - - build-kube-ovn-dpdk-base: - name: Build kube-ovn-dpdk-base - runs-on: ubuntu-22.04 - outputs: - build-dpdk-base: ${{ steps.check.outputs.build-dpdk-base }} - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - id: check - run: | - if [ ${{ github.event_name }} != 'pull_request' ]; then - exit - fi - if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base-dpdk$; then - echo build-dpdk-base=1 >> "$GITHUB_OUTPUT" - fi - - - uses: docker/setup-buildx-action@v3 - if: steps.check.outputs.build-dpdk-base == 1 - - - name: Build kube-ovn-dpdk-base image - id: build - if: steps.check.outputs.build-dpdk-base == 1 - run: | - make base-amd64-dpdk - make base-tar-amd64-dpdk - - - name: Upload dpdk base images to artifact - if: steps.check.outputs.build-dpdk-base == 1 - uses: actions/upload-artifact@v4 - with: - name: kube-ovn-dpdk-base - path: image-amd64-dpdk.tar - build-kube-ovn: name: Build kube-ovn runs-on: ubuntu-22.04 - needs: - - build-kube-ovn-base - - build-kube-ovn-dpdk-base steps: - uses: jlumbroso/free-disk-space@v1.3.1 with: @@ -207,196 +112,17 @@ jobs: fi make tar-kube-ovn - - name: Build dpdk - run: | - if [ ${{ needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base || 0 }} = 1 ]; then - TAG=$(cat VERSION) - docker tag kubeovn/kube-ovn-base:$TAG-amd64-dpdk kubeovn/kube-ovn-base:$TAG-dpdk - make build-kube-ovn-dpdk - else - make image-kube-ovn-dpdk - fi - make tar-kube-ovn-dpdk - - name: Upload images to artifact uses: actions/upload-artifact@v4 with: name: kube-ovn path: kube-ovn.tar - - name: Upload dpdk images to artifact - uses: actions/upload-artifact@v4 - if: github.event_name != 'pull_request' - with: - name: kube-ovn-dpdk - path: kube-ovn-dpdk.tar - - build-vpc-nat-gateway: - name: Build vpc-nat-gateway - runs-on: ubuntu-22.04 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Build - run: | - make image-vpc-nat-gateway - make tar-vpc-nat-gateway - - - name: Upload image to artifact - uses: actions/upload-artifact@v4 - with: - name: vpc-nat-gateway - path: vpc-nat-gateway.tar - - build-centos-compile: - name: Build centos-compile - runs-on: ubuntu-22.04 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - - name: Build - run: | - make image-centos-compile - make tar-centos-compile - - - name: Upload centos7-compile image to artifact - uses: actions/upload-artifact@v4 - with: - name: centos7-compile - path: centos7-compile.tar - - # - name: Upload centos8-compile image to artifact - # uses: actions/upload-artifact@v4 - # with: - # name: centos8-compile - # path: centos8-compile.tar - - build-e2e-binaries: - name: Build E2E Binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - run: make e2e-build - working-directory: ${{ env.E2E_DIR }} - - netpol-path-filter: - name: Network Policy Path Filter - if: github.event_name != 'pull_request' - runs-on: ubuntu-22.04 - outputs: - test-netpol: ${{ steps.filter.outputs.kube-ovn-controller }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: go.mod - check-latest: true - cache: false - - - name: Generate path filter - run: | - filter=".github/path-filters.yaml" - cat > $filter <> "$GITHUB_ENV" fi - - name: Remove DNS search domain - run: | - sudo sed -i '/^search/d' /etc/resolv.conf - sudo systemctl restart docker - - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION || '' }} @@ -481,1870 +211,20 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar - - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" - - - name: Create kind cluster + - name: Create kind clusters run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }} + sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic-${{ matrix.ip-family }} sudo cp -r /root/.kube/ ~/.kube/ sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind - run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }} + run: make kind-install-ovn-ic-${{ matrix.ip-family }} - name: Run E2E working-directory: ${{ env.E2E_DIR }} env: E2E_BRANCH: ${{ github.base_ref || github.ref_name }} E2E_IP_FAMILY: ${{ matrix.ip-family }} - E2E_NETWORK_MODE: ${{ matrix.mode }} - run: make k8s-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log - path: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log.tar.gz - - - name: Check valgrind result - run: | - kubectl -n kube-system rollout restart deploy ovn-central - kubectl -n kube-system rollout restart ds ovs-ovn - kubectl -n kube-system rollout status deploy ovn-central - kubectl -n kube-system rollout status ds ovs-ovn - while true; do - if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then - break - fi - sleep 1 - done - kubectl ko log ovn - kubectl ko log ovs - - for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do - echo "Checking if valgrind log file for $daemon exists..." - find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 - done - - find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do - if grep -qw 'definitely lost' "$f"; then - echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." - echo $f - cat "$f" - exit 1 - fi; - done - - k8s-netpol-e2e: - name: Kubernetes Network Policy E2E - if: | - always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && - (needs.netpol-path-filter.outputs.test-netpol == 1 || contains(github.event.pull_request.labels.*.name, 'network policy')) - needs: - - build-kube-ovn - - build-e2e-binaries - - netpol-path-filter - runs-on: ubuntu-22.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - ip-family: - - ipv4 - - ipv6 - - dual - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - name: Remove DNS search domain - run: | - sudo sed -i '/^search/d' /etc/resolv.conf - sudo systemctl restart docker - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind - run: make kind-install-${{ matrix.ip-family }} - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - run: make k8s-netpol-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log - path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: Check valgrind result - run: | - kubectl -n kube-system rollout restart deploy ovn-central - kubectl -n kube-system rollout restart ds ovs-ovn - kubectl -n kube-system rollout status deploy ovn-central - kubectl -n kube-system rollout status ds ovs-ovn - while true; do - if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then - break - fi - sleep 1 - done - kubectl ko log ovn - kubectl ko log ovs - - for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do - echo "Checking if valgrind log file for $daemon exists..." - find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 - done - - find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do - if grep -qw 'definitely lost' "$f"; then - echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." - echo $f - cat "$f" - exit 1 - fi; - done - - cyclonus-netpol-e2e: - name: Cyclonus Network Policy E2E - if: | - always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && - (needs.netpol-path-filter.outputs.test-netpol == 1 || contains(github.event.pull_request.labels.*.name, 'network policy')) - needs: - - build-kube-ovn - - netpol-path-filter - runs-on: ubuntu-22.04 - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - ip-family: - - ipv4 - - ipv6 - - dual - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind - run: make kind-install-${{ matrix.ip-family }} - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - run: make cyclonus-netpol-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log - path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: Check valgrind result - run: | - kubectl -n kube-system rollout restart deploy ovn-central - kubectl -n kube-system rollout restart ds ovs-ovn - kubectl -n kube-system rollout status deploy ovn-central - kubectl -n kube-system rollout status ds ovs-ovn - while true; do - if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then - break - fi - sleep 1 - done - kubectl ko log ovn - kubectl ko log ovs - - for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do - echo "Checking if valgrind log file for $daemon exists..." - find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 - done - - find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do - if grep -qw 'definitely lost' "$f"; then - echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." - echo $f - cat "$f" - exit 1 - fi; - done - - kube-ovn-conformance-e2e: - name: Kube-OVN Conformance E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - ip-family: - - ipv4 - - ipv6 - - dual - mode: - - overlay - - underlay - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind - run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }} - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - E2E_IP_FAMILY: ${{ matrix.ip-family }} - E2E_NETWORK_MODE: ${{ matrix.mode }} - run: make kube-ovn-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log - path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: Check valgrind result - run: | - kubectl -n kube-system rollout restart deploy ovn-central - kubectl -n kube-system rollout restart ds ovs-ovn - kubectl -n kube-system rollout status deploy ovn-central - kubectl -n kube-system rollout status ds ovs-ovn - while true; do - if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then - break - fi - sleep 1 - done - kubectl ko log ovn - kubectl ko log ovs - - for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do - echo "Checking if valgrind log file for $daemon exists..." - find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 - done - - find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do - if grep -qw 'definitely lost' "$f"; then - echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." - echo $f - cat "$f" - exit 1 - fi; - done - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - kube-ovn-ic-conformance-e2e: - name: Kube-OVN IC Conformance E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - ip-family: - - ipv4 - - ipv6 - - dual - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind clusters - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install-ovn-ic-${{ matrix.ip-family }} - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - E2E_IP_FAMILY: ${{ matrix.ip-family }} - run: make kube-ovn-ic-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - for cluster in `kind get clusters`; do - kubectl config use-context kind-$cluster - make kubectl-ko-log - mv kubectl-ko-log.tar.gz kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-$cluster-ko-log.tar.gz - done - tar zcvf kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz \ - `kind get clusters | xargs -I {} echo kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-{}-ko-log.tar.gz` - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log - path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz - - chart-test: - name: Chart Installation/Uninstallation Test - needs: build-kube-ovn - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - ssl: - - "true" - - "false" - timeout-minutes: 30 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - ENABLE_SSL: "${{ matrix.ssl }}" - run: make kind-install-chart - - - name: Uninstall Kube-OVN - run: make kind-uninstall-chart - - underlay-logical-gateway-installation-test: - name: Underlay Logical Gateway Installation Test - needs: build-kube-ovn - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-dual - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install-underlay-logical-gateway-dual - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - no-ovn-lb-test: - name: Disable OVN LB Test - needs: build-kube-ovn - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN without LoadBalancer - env: - ENABLE_LB: "false" - run: make kind-install - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - no-np-test: - name: Disable Network Policy Test - needs: build-kube-ovn - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - ENABLE_NP: "false" - run: make kind-install - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - lb-svc-e2e: - name: LB Service E2E - needs: - - build-kube-ovn - - build-vpc-nat-gateway - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Download vpc-nat-gateway image - uses: actions/download-artifact@v4 - with: - name: vpc-nat-gateway - - - name: Load images - run: | - docker load -i kube-ovn.tar - docker load -i vpc-nat-gateway.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Multus and Kube-OVN - run: make kind-install-lb-svc - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make kube-ovn-lb-svc-conformance-e2e - - kubevirt-e2e: - name: Kubevirt VM E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load images - run: | - docker load -i kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN and KubeVirt - run: make kind-install-kubevirt - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make kube-ovn-kubevirt-e2e - - webhook-e2e: - name: Webhook E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load images - run: | - docker load -i kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install-webhook - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make kube-ovn-webhook-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz webhook-e2e-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: webhook-e2e-ko-log - path: webhook-e2e-ko-log.tar.gz - - installation-compatibility-test: - name: Installation Compatibility Test - needs: build-kube-ovn - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH k8s_version=v1.23.17 make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz installation-compatibility-test-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: installation-compatibility-test-ko-log - path: installation-compatibility-test-ko-log.tar.gz - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - cilium-chaining-e2e: - name: Cilium Chaining E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - ip-family: - - ipv4 - # - ipv6 - # - dual - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - uses: azure/setup-helm@v4.0.0 - with: - version: '${{ env.HELM_VERSION }}' - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - name: Remove DNS search domain - run: | - sudo sed -i '/^search/d' /etc/resolv.conf - sudo systemctl restart docker - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-cilium-chaining-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN with Cilium chaining - run: make kind-install-cilium-chaining-${{ matrix.ip-family }} - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_CILIUM_CHAINING: "true" - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - E2E_IP_FAMILY: ${{ matrix.ip-family }} - run: make k8s-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz cilium-chaining-e2e-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: cilium-chaining-e2e-ko-log - path: cilium-chaining-e2e-ko-log.tar.gz - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - kube-ovn-ha-e2e: - name: Kube-OVN HA E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - ssl: - - "true" - - "false" - bind-local: - - "true" - - "false" - ip-family: - - ipv4 - - ipv6 - - dual - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-ha-${{ matrix.ip-family }} - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - env: - NET_STACK: "${{ matrix.ip-family }}" - ENABLE_SSL: "${{ matrix.ssl }}" - ENABLE_BIND_LOCAL_IP: "${{ matrix.bind-local }}" - run: make kind-install-chart - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - E2E_IP_FAMILY: ${{ matrix.ip-family }} - run: | - make kube-ovn-security-e2e - make kube-ovn-ha-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log - path: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - kube-ovn-submariner-conformance-e2e: - name: Kube-OVN Submariner Conformance E2E - needs: - - build-kube-ovn - runs-on: ubuntu-22.04 - timeout-minutes: 60 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install submariner subctl - env: - VERSION: v${{ env.SUBMARINER_VERSION }} - DESTDIR: /usr/local/bin - run: curl -Ls https://get.submariner.io | bash - - - name: Download image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load image - run: docker load --input kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-ovn-submariner - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install-ovn-submariner - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - run: make kube-ovn-submariner-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz kube-ovn-submariner-conformance-e2e-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: kube-ovn-submariner-conformance-e2e-ko-log - path: kube-ovn-submariner-conformance-e2e-ko-log.tar.gz - - - name: Cleanup - run: sh -x dist/images/cleanup.sh - - iptables-vpc-nat-gw-conformance-e2e: - name: Iptables VPC NAT Gateway E2E - needs: - - build-kube-ovn - - build-vpc-nat-gateway - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Download vpc-nat-gateway image - uses: actions/download-artifact@v4 - with: - name: vpc-nat-gateway - - - name: Load images - run: | - docker load -i kube-ovn.tar - docker load -i vpc-nat-gateway.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN with VPC NAT gateway enabled - run: make kind-install-vpc-nat-gw - - - name: Run E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make iptables-vpc-nat-gw-conformance-e2e - - - name: kubectl ko log - if: failure() - run: | - make kubectl-ko-log - mv kubectl-ko-log.tar.gz iptables-vpc-nat-gw-conformance-e2e-ko-log.tar.gz - - - name: upload kubectl ko log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: iptables-vpc-nat-gw-conformance-e2e-ko-log - path: iptables-vpc-nat-gw-conformance-e2e-ko-log.tar.gz - - ovn-vpc-nat-gw-conformance-e2e: - name: OVN VPC NAT Gateway E2E - needs: - - build-kube-ovn - - build-e2e-binaries - runs-on: ubuntu-22.04 - timeout-minutes: 15 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Create the default branch directory - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - run: mkdir -p test/e2e/source - - - name: Check out the default branch - if: (github.base_ref || github.ref_name) != github.event.repository.default_branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - fetch-depth: 1 - path: test/e2e/source - - - name: Export E2E directory - run: | - if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then - echo "E2E_DIR=." >> "$GITHUB_ENV" - else - echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" - fi - - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION || '' }} - go-version-file: ${{ env.E2E_DIR }}/go.mod - check-latest: true - cache: false - - - name: Export Go full version - run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" - - - name: Go cache - uses: actions/cache/restore@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} - restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - - - name: Install kind - uses: helm/kind-action@v1.9.0 - with: - version: ${{ env.KIND_VERSION }} - install_only: true - - - name: Install ginkgo - working-directory: ${{ env.E2E_DIR }} - run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Load images - run: docker load -i kube-ovn.tar - - - name: Create kind cluster - run: | - sudo pip3 install j2cli - sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init - sudo cp -r /root/.kube/ ~/.kube/ - sudo chown -R $(id -un). ~/.kube/ - - - name: Install Kube-OVN - run: make kind-install - - - name: Run Vip E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make vip-conformance-e2e - - - name: Run Ovn VPC NAT GW E2E - working-directory: ${{ env.E2E_DIR }} - env: - E2E_BRANCH: ${{ github.base_ref || github.ref_name }} - run: make ovn-vpc-nat-gw-conformance-e2e - - push: - name: Push Images - needs: - - build-centos-compile - - k8s-conformance-e2e - - k8s-netpol-e2e - - cyclonus-netpol-e2e - - kube-ovn-conformance-e2e - - kube-ovn-ic-conformance-e2e - - ovn-vpc-nat-gw-conformance-e2e - - iptables-vpc-nat-gw-conformance-e2e - - webhook-e2e - - lb-svc-e2e - - underlay-logical-gateway-installation-test - - chart-test - - installation-compatibility-test - - no-ovn-lb-test - - no-np-test - - cilium-chaining-e2e - - kube-ovn-ha-e2e - - kubevirt-e2e - - kube-ovn-submariner-conformance-e2e - if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') - runs-on: ubuntu-22.04 - steps: - - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - haskell: true - docker-images: false - large-packages: false - tool-cache: false - swap-storage: false - - - uses: actions/checkout@v4 - - - name: Download kube-ovn image - uses: actions/download-artifact@v4 - with: - name: kube-ovn - - - name: Download kube-ovn-dpdk image - uses: actions/download-artifact@v4 - if: github.event_name != 'pull_request' - with: - name: kube-ovn-dpdk - - - name: Download vpc-nat-gateway image - uses: actions/download-artifact@v4 - with: - name: vpc-nat-gateway - - - name: Download centos7-compile image - uses: actions/download-artifact@v4 - with: - name: centos7-compile - - # - name: Download centos8-compile image - # uses: actions/download-artifact@v4 - # with: - # name: centos8-compile - - - name: Load image - run: | - docker load --input kube-ovn.tar - docker load --input vpc-nat-gateway.tar - docker load --input centos7-compile.tar - # docker load --input centos8-compile.tar - if [ '${{ github.event_name }}' != 'pull_request' ]; then - docker load --input kube-ovn-dpdk.tar - fi - - - name: Security Scan - run: | - sudo apt-get install wget apt-transport-https gnupg lsb-release - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - - echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - make scan - - - name: Push - if: github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, 'release-') - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - COMMIT: ${{ github.sha }} - run: | - cat VERSION - TAG=$(cat VERSION) - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn-dev:$COMMIT-x86 - docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn:$TAG-x86 - docker tag kubeovn/kube-ovn:$TAG-debug kubeovn/kube-ovn:$TAG-debug-x86 - docker tag kubeovn/kube-ovn:$TAG-dpdk kubeovn/kube-ovn:$TAG-dpdk-x86 - docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway-dev:$COMMIT-x86 - docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway:$TAG-x86 - docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile-dev:$TAG-x86 - docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile:$TAG-x86 - # docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile-dev:$TAG-x86 - # docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile:$TAG-x86 - docker images - docker push kubeovn/kube-ovn:$TAG-x86 - docker push kubeovn/kube-ovn-dev:$COMMIT-x86 - docker push kubeovn/kube-ovn:$TAG-debug-x86 - docker push kubeovn/kube-ovn:$TAG-dpdk-x86 - docker push kubeovn/vpc-nat-gateway:$TAG-x86 - docker push kubeovn/vpc-nat-gateway-dev:$COMMIT-x86 - docker push kubeovn/centos7-compile:$TAG-x86 - docker push kubeovn/centos7-compile-dev:$TAG-x86 - # docker push kubeovn/centos8-compile:$TAG-x86 - # docker push kubeovn/centos8-compile-dev:$TAG-x86 + run: make kube-ovn-ic-conformance-e2e diff --git a/.gitignore b/.gitignore index 85e87a60aaf..1ba5fa61ed7 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ kube-ovn-cni-sa.yaml kube-ovn-sa.yaml ovn-ovs-sa.yaml ovs-ovn-ds.yaml +ovn-ic-controller.yaml +ovn-ic-server.yaml diff --git a/Makefile b/Makefile index bb4316b9c18..26041d76175 100644 --- a/Makefile +++ b/Makefile @@ -359,7 +359,7 @@ kind-init-ovn-ic: kind-init-ovn-ic-ipv4 .PHONY: kind-init-ovn-ic-% kind-init-ovn-ic-%: kind-clean-ovn-ic - @ha=true $(MAKE) kind-init-$* + @ovn_ic=true $(MAKE) kind-init-$* @ovn_ic=true ip_family=$* $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) @@ -977,6 +977,8 @@ clean: $(RM) yamls/kind.yaml $(RM) yamls/clab-bgp.yaml yamls/clab-bgp-ha.yaml $(RM) ovn.yaml kube-ovn.yaml kube-ovn-crd.yaml + $(RM) kube-ovn-app-sa.yaml kube-ovn-cni-sa.yaml kube-ovn-sa.yaml ovn-ovs-sa.yaml + $(RM) ovs-ovn-ds.yaml ovn-ic-controller.yaml ovn-ic-server.yaml $(RM) ovn-ic-0.yaml ovn-ic-1.yaml $(RM) kwok-node.yaml $(RM) kube-ovn.tar kube-ovn-dpdk.tar vpc-nat-gateway.tar image-amd64.tar image-amd64-dpdk.tar image-arm64.tar diff --git a/Makefile.e2e b/Makefile.e2e index 8b54e3329ec..903f0a4c298 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -123,7 +123,7 @@ kube-ovn-ic-conformance-e2e: E2E_BRANCH=$(E2E_BRANCH) \ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ - ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + ginkgo --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/ovn-ic/ovn-ic.test -- $(TEST_BIN_ARGS) .PHONY: kube-ovn-submariner-conformance-e2e diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index 2c9b4d937a0..67bad6d029c 100755 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -2,7 +2,7 @@ set -eo pipefail LOCAL_IP=${LOCAL_IP:-$POD_IP} -TS_NUM=${TS_NUM:-ts} +TS_NUM=${TS_NUM:-3} ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true} ENABLE_OVN_LEADER_CHECK=${ENABLE_OVN_LEADER_CHECK:-true} @@ -213,7 +213,7 @@ fi if [[ $ENABLE_OVN_LEADER_CHECK == "true" ]]; then chmod 600 /etc/ovn/* - /kube-ovn/kube-ovn-leader-checker --probeInterval=${OVN_LEADER_PROBE_INTERVAL} --isICDBServer=true + TS_NUM=${TS_NUM} /kube-ovn/kube-ovn-leader-checker --probeInterval=${OVN_LEADER_PROBE_INTERVAL} --isICDBServer=true else # Compatible with controller deployment methods before kube-ovn 1.11.16 TS_NAME=${TS_NAME:-ts} diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index 92c875079f5..6e121029ca4 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -8,6 +8,7 @@ import ( "math/rand/v2" "net" "os/exec" + "slices" "strconv" "strings" "testing" @@ -51,6 +52,7 @@ func TestE2E(t *testing.T) { if len(clusters) < 2 { t.Fatal("no enough kind clusters to run ovn-ic e2e testing") } + slices.Sort(clusters) k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) e2e.RunE2ETests(t) @@ -80,12 +82,14 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { clientSets := make([]clientset.Interface, len(clusters)) podClients := make([]*framework.PodClient, len(clusters)) + deployClients := make([]*framework.DeploymentClient, len(clusters)) namespaceNames := make([]string, len(clusters)) var kubectlConfig string ginkgo.BeforeEach(func() { for i := range clusters { clientSets[i] = frameworks[i].ClientSet podClients[i] = frameworks[i].PodClient() + deployClients[i] = frameworks[i].DeploymentClientNS(framework.KubeOvnNamespace) namespaceNames[i] = frameworks[i].Namespace.Name } kubectlConfig = k8sframework.TestContext.KubeConfig @@ -139,6 +143,11 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { } } } + + for i := range clusters { + ginkgo.By("Deleting pod " + podNames[i] + " in cluster " + clusters[i]) + framework.ExpectNoError(podClients[i].Delete(podNames[i])) + } } framework.ConformanceIt("should create logical switch ts", func() { @@ -165,11 +174,11 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { ginkgo.By("case 2: Delete configmap ovn-ic-config and rebuild it") execCmd := "kubectl get configmap ovn-ic-config -n kube-system -oyaml > temp-ovn-ic-config.yaml; kubectl delete configmap ovn-ic-config -n kube-system" - _, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + _, err := exec.Command("bash", "-ec", execCmd).CombinedOutput() framework.ExpectNoError(err) execCmd = "kubectl apply -f temp-ovn-ic-config.yaml; rm -f temp-ovn-ic-config.yaml" - _, err = exec.Command("bash", "-c", execCmd).CombinedOutput() + _, err = exec.Command("bash", "-ec", execCmd).CombinedOutput() framework.ExpectNoError(err) fnCheckPodHTTP() }) @@ -222,11 +231,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ConformanceIt("Should Support ECMP OVN Interconnection", func() { frameworks[0].SkipVersionPriorTo(1, 11, "This feature was introduced in v1.11") ginkgo.By("case 1: ecmp gateway network test") - if frameworks[0].ClusterIPFamily == "dual" { - checkECMPCount(6) - } else { - checkECMPCount(3) - } + checkECMPCount(frameworks, 3) fnCheckPodHTTP() ginkgo.By("case 2: reduce two clusters from 3 gateway to 1 gateway") @@ -267,96 +272,83 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() ginkgo.By("case 4: scale ecmp path from 3 to 5") - switchCmd := "kubectl config use-context kind-kube-ovn" - _, err := exec.Command("bash", "-c", switchCmd).CombinedOutput() - framework.ExpectNoError(err, "switch to kube-ovn cluster failed") - - patchCmd := "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '5'}]\"" - _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() - if frameworks[0].ClusterIPFamily == "dual" { - checkECMPCount(10) - } else { - checkECMPCount(5) + icServer := deployClients[0].Get("ovn-ic-server") + patchedICServer := icServer.DeepCopy() + for i, env := range patchedICServer.Spec.Template.Spec.Containers[0].Env { + if env.Name == "TS_NUM" { + patchedICServer.Spec.Template.Spec.Containers[0].Env[i].Value = "5" + } } + icServer = deployClients[0].PatchSync(icServer, patchedICServer) + checkECMPCount(frameworks, 5) fnCheckPodHTTP() ginkgo.By("case 5: reduce ecmp path from 5 to 3") - patchCmd = "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '3'}]\"" - _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() - if frameworks[0].ClusterIPFamily == "dual" { - checkECMPCount(6) - } else { - checkECMPCount(3) + patchedICServer = icServer.DeepCopy() + for i, env := range patchedICServer.Spec.Template.Spec.Containers[0].Env { + if env.Name == "TS_NUM" { + patchedICServer.Spec.Template.Spec.Containers[0].Env[i].Value = "3" + } } + _ = deployClients[0].PatchSync(icServer, patchedICServer) + checkECMPCount(frameworks, 3) fnCheckPodHTTP() ginkgo.By("case 6: disable gateway kube-ovn1-worker gateway") - switchCmd = "kubectl config use-context kind-kube-ovn1" - _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() - framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") - disableNetworkCmd := "docker exec kube-ovn1-worker iptables -I INPUT -p udp --dport 6081 -j DROP" - _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + _, err := exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() framework.ExpectNoError(err, "disable kube-ovn1-worker gateway failed") - taintCmd := "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule" - _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + taintCmd := "taint nodes kube-ovn1-worker e2e=test:NoSchedule" + execOrDie("kind-kube-ovn1", taintCmd) fnCheckPodHTTP() ginkgo.By("case 7: disable gateway kube-ovn1-worker2 gateway") - switchCmd = "kubectl config use-context kind-kube-ovn1" - _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() - framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") - disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -I INPUT -p udp --dport 6081 -j DROP" _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() framework.ExpectNoError(err, "disable kube-ovn1-worker2 gateway failed") - taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule" - _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + taintCmd = "taint nodes kube-ovn1-worker2 e2e=test:NoSchedule" + execOrDie("kind-kube-ovn1", taintCmd) fnCheckPodHTTP() ginkgo.By("case 8: enable gateway kube-ovn1-worker gateway") - switchCmd = "kubectl config use-context kind-kube-ovn1" - _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() - framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") - disableNetworkCmd = "docker exec kube-ovn1-worker iptables -D INPUT -p udp --dport 6081 -j DROP" _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() framework.ExpectNoError(err, "enable kube-ovn1-worker gateway failed") - taintCmd = "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule-" - _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + taintCmd = "taint nodes kube-ovn1-worker e2e=test:NoSchedule-" + execOrDie("kind-kube-ovn1", taintCmd) fnCheckPodHTTP() ginkgo.By("case 9: enable gateway kube-ovn1-worker2 gateway") - switchCmd = "kubectl config use-context kind-kube-ovn1" - _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() - framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") - disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -D INPUT -p udp --dport 6081 -j DROP" _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() framework.ExpectNoError(err, "enable kube-ovn1-worker2 gateway failed") - taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule-" - _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + taintCmd = "taint nodes kube-ovn1-worker2 e2e=test:NoSchedule-" + execOrDie("kind-kube-ovn1", taintCmd) fnCheckPodHTTP() }) }) -func checkECMPCount(expectCount int) { +func checkECMPCount(frameworks []*framework.Framework, expectedCount int) { + if frameworks[0].ClusterIPFamily == framework.Dual { + expectedCount *= 2 + } + ecmpCount := 0 maxRetryTimes := 30 - for i := 0; i < maxRetryTimes; i++ { - time.Sleep(3 * time.Second) - execCmd := "kubectl ko nbctl lr-route-list ovn-cluster " - output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() - framework.ExpectNoError(err) - ecmpCount = strings.Count(string(output), "ecmp") - if ecmpCount == expectCount { - break + var output string + for _, f := range frameworks { + for i := 0; i < maxRetryTimes; i++ { + time.Sleep(3 * time.Second) + output = execOrDie(f.KubeContext, "ko nbctl lr-route-list ovn-cluster") + ecmpCount = strings.Count(output, "ecmp") + if ecmpCount == expectedCount { + break + } } + framework.ExpectEqual(ecmpCount, expectedCount, "logical router routes: %v", output) } - - framework.ExpectEqual(ecmpCount, expectCount) }