github: switch to use lxc (#603)

vagrant on macos is fragile recently, try to use lxc. * remove obsolete condition from install-newly.sh since v5.0.2, user/group is changed to be kept. * github: added AmazonLinux, note that ubuntu-latest has switched to CGroup V2, so can't launch CGroup V1 container by default.
fluent · Dec 8, 2023 · c3463c8 · c3463c8
1 parent b793ab1
commit c3463c8
Show file tree

Hide file tree

Showing 14 changed files with 146 additions and 101 deletions.
diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml
@@ -95,22 +95,36 @@ jobs:
   test:
     name: Test
     needs: build
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        distribution:
-          - debian-bullseye
-          - debian-bookworm
-          - ubuntu-focal
-          - ubuntu-jammy
+        label:
+          - Debian GNU/Linux bullseye amd64
+          - Debian GNU/Linux bookworm amd64
+          - Ubuntu Focal amd64
+          - Ubuntu Jammy amd64
+        include:
+          - label: Debian GNU/Linux bullseye amd64
+            rake-job: debian-bullseye
+            test-lxc-image: images:debian/11
+          - label: Debian GNU/Linux bookworm amd64
+            rake-job: debian-bookworm
+            test-lxc-image: images:debian/12
+          - label: Ubuntu Focal amd64
+            rake-job: ubuntu-focal
+            test-lxc-image: ubuntu:20.04
+          - label: Ubuntu Jammy amd64
+            rake-job: ubuntu-jammy
+            test-lxc-image: ubuntu:22.04
     steps:
       - uses: actions/checkout@master
       - uses: actions/download-artifact@master
         with:
-          name: packages-${{ matrix.distribution }}
+          name: packages-${{ matrix.rake-job }}
       - uses: actions/download-artifact@master
         with:
-          name: packages-apt-source-${{ matrix.distribution }}
-      - name: Run Test
-        run: fluent-package/apt/systemd-test/test.sh ${{ matrix.distribution }}
+          name: packages-apt-source-${{ matrix.rake-job }}
+      - uses: canonical/setup-lxd@v0.1.1
+      - name: Run Test 
+        run: fluent-package/apt/systemd-test/test.sh ${{ matrix.test-lxc-image }}
diff --git a/.github/workflows/yum.yml b/.github/workflows/yum.yml
@@ -89,23 +89,58 @@ jobs:
           --env CENTOS_STREAM=${{ matrix.centos-stream }} \
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/binstubs-test.sh
-  test:
-    name: Test
+  v1test:
+    name: Test with CGroup V1
     needs: build
-    runs-on: macos-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
-        distribution:
-          - centos-7
-          - rockylinux-8
-          - almalinux-9
-          - amazonlinux-2
-          # - amazonlinux-2023
+        label:
+          - CentOS 7 x86_64
+          - AmazonLinux 2 x86_64
+          - AmazonLinux 2023 x86_64
+        include:
+          - label: CentOS 7 x86_64
+            rake-job: centos-7
+            test-lxc-image: images:centos/7
+          - label: AmazonLinux 2 x86_64
+            rake-job: amazonlinux-2
+            test-lxc-image: images:amazonlinux/2
+          - label: AmazonLinux 2023 x86_64
+            rake-job: amazonlinux-2023
+            test-lxc-image: images:amazonlinux/2023
     steps:
       - uses: actions/checkout@master
       - uses: actions/download-artifact@master
         with:
-          name: packages-${{ matrix.distribution }}
+          name: packages-${{ matrix.rake-job }}
+      - uses: canonical/setup-lxd@v0.1.1
+      - name: Run Test
+        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }}
+
+  v2test:
+    name: Test with CGroup V2
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        label:
+          - RockyLinux 8 x86_64
+          - AlmaLinux 9 x86_64
+        include:
+          - label: RockyLinux 8 x86_64
+            rake-job: rockylinux-8
+            test-lxc-image: images:rockylinux/8
+          - label: AlmaLinux 9 x86_64
+            rake-job: almalinux-9
+            test-lxc-image: images:almalinux/9
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/download-artifact@master
+        with:
+          name: packages-${{ matrix.rake-job }}
+      - uses: canonical/setup-lxd@v0.1.1
       - name: Run Test
-        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.distribution }}
+        run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }}
diff --git a/fluent-package/apt/systemd-test/install-newly.sh b/fluent-package/apt/systemd-test/install-newly.sh
@@ -7,7 +7,7 @@ set -exu
 case $1 in
   local)
     sudo apt install -V -y \
-      /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+      /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
     ;;
   v5)
     curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5.sh | sh

diff --git a/fluent-package/apt/systemd-test/setup.sh b/fluent-package/apt/systemd-test/setup.sh
@@ -3,4 +3,4 @@
 set -exu
 
 sudo apt update
-sudo apt install -V -y lsb-release
+sudo apt install -V -y lsb-release curl
diff --git a/fluent-package/apt/systemd-test/test.sh b/fluent-package/apt/systemd-test/test.sh
@@ -1,28 +1,21 @@
 #!/bin/bash
 
 if [ -z $1 ]; then
-    echo "Error: Need to specify VM name in the Vagrantfile."
-    echo "Ex.) $ ./test.sh ubuntu-focal"
+    echo "Error: Need to specify lxc image name."
+    echo "Ex.) $ ./test.sh ubuntu:20.04"
     exit 1
 fi
 
-vm=$1
-dir="/vagrant/fluent-package/apt/systemd-test"
+image=$1
+dir="/host/fluent-package/apt/systemd-test"
 
-vagrant status $vm | grep -E "^${vm}\s+not created (.*)$"
-if [ $? -ne 0 ]; then
-    echo "Error: The VM already exists. Need to destroy it in advance with the following command."
-    echo "$ vagrant destroy $vm"
-    exit 1
-fi
-
-set -eu
+set -eux
 
 test_filenames=(
     update-to-next-version.sh
 )
 
-if [ ! $vm = "debian-bookworm" ]; then
+if [ ! $image = "images:debian/12" ]; then
     # As no bookworm package for v4, so execute upgrade test for other code name.
     test_filenames+=(
         update-from-v4.sh
@@ -32,24 +25,26 @@ fi
 
 for apt_repo_type in local v5 lts; do
     echo -e "\nRun test: $apt_repo_type\n"
-    vagrant up $vm
-    vagrant ssh $vm -- $dir/setup.sh
-    vagrant ssh $vm -- $dir/install-newly.sh $apt_repo_type
-    vagrant destroy -f $vm
+    lxc launch $image target
+    sleep 5
+    lxc config device add target host disk source=$PWD path=/host
+    lxc list
+    lxc exec target -- $dir/setup.sh
+    lxc exec target -- $dir/install-newly.sh $apt_repo_type
+    lxc stop target
+    lxc delete target
 done
 
 for test_filename in ${test_filenames[@]}; do
     echo -e "\nRun test: $test_filename\n"
-    vagrant up $vm
-    vagrant ssh $vm -- $dir/setup.sh
-    vagrant ssh $vm -- $dir/$test_filename
-    vagrant destroy -f $vm
-    # I want to use snapshot instead of destorying it for every test,
-    # but somehow, it will be often an error on GitHub Actions...
-    #   $ vagrant ssh $vm -- $dir/setup.sh
-    #   $ vagrant snapshot save -f $vm after-setup
-    #   (execute a test)
-    #   $ vagrant snapshot restore $vm after-setup
+    lxc launch $image target
+    sleep 5
+    lxc config device add target host disk source=$PWD path=/host
+    lxc list
+    lxc exec target -- $dir/setup.sh
+    lxc exec target -- $dir/$test_filename
+    lxc stop target
+    lxc delete target
 done
 
 echo -e "\nAll Success!\n"
diff --git a/fluent-package/apt/systemd-test/update-from-v4.sh b/fluent-package/apt/systemd-test/update-from-v4.sh
@@ -12,7 +12,7 @@ sudo apt install -y ./fluentd-apt-source_2020.8.25-1_all.deb
 # Install v4
 sudo apt clean all
 # Uncomment when v5 repository was deployed
-#apt_source_package=/vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-apt-source*_all.deb
+#apt_source_package=/host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-apt-source*_all.deb
 #sudo apt install -V -y ${apt_source_package} ca-certificates
 sudo apt update
 sudo apt install -V -y td-agent=${td_agent_version}-1
@@ -21,7 +21,7 @@ systemctl status --no-pager td-agent
 
 # Install the current
 sudo apt install -V -y \
-    /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
 
 # Test: service status
 systemctl status --no-pager fluentd

diff --git a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh
@@ -12,7 +12,7 @@ sudo apt install -y ./fluentd-apt-source_2020.8.25-1_all.deb
 # Install v4
 sudo apt clean all
 # Uncomment when v5 repository was deployed
-#apt_source_package=/vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-apt-source*_all.deb
+#apt_source_package=/host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-apt-source*_all.deb
 #sudo apt install -V -y ${apt_source_package} ca-certificates
 sudo apt update
 sudo apt install -V -y td-agent=${td_agent_version}-1
@@ -21,7 +21,7 @@ systemctl status --no-pager td-agent
 
 # Install the current
 sudo apt install -V -y \
-    /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
 systemctl status --no-pager fluentd
 
 sudo systemctl stop fluentd
@@ -32,7 +32,7 @@ systemctl status --no-pager fluentd
 systemctl status --no-pager td-agent
 
 # Make a dummy pacakge for the next version
-dpkg-deb -R /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
+dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
 last_ver=$(cat tmp/DEBIAN/control | grep "Version: " | sed -E "s/Version: ([0-9.]+)-([0-9]+)/\2/g")
 sed -i -E "s/Version: ([0-9.]+)-([0-9]+)/Version: \1-$(($last_ver+1))/g" tmp/DEBIAN/control
 dpkg-deb --build tmp next_version.deb

diff --git a/fluent-package/apt/systemd-test/update-to-next-version.sh b/fluent-package/apt/systemd-test/update-to-next-version.sh
@@ -6,10 +6,10 @@ set -exu
 
 # Install the current
 sudo apt install -V -y \
-    /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
 
 # Make a dummy pacakge for the next version
-dpkg-deb -R /vagrant/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
+dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
 last_ver=$(cat tmp/DEBIAN/control | grep "Version: " | sed -E "s/Version: ([0-9.]+)-([0-9]+)/\2/g")
 sed -i -E "s/Version: ([0-9.]+)-([0-9]+)/Version: \1-$(($last_ver+1))/g" tmp/DEBIAN/control
 dpkg-deb --build tmp next_version.deb

diff --git a/fluent-package/yum/systemd-test/commonvar.sh b/fluent-package/yum/systemd-test/commonvar.sh
@@ -2,7 +2,7 @@
 
 distribution=$(cat /etc/system-release-cpe | awk '{print substr($0, index($1, "o"))}' | cut -d: -f2)
 version=$(cat /etc/system-release-cpe | awk '{print substr($0, index($1, "o"))}' | cut -d: -f4)
-td_agent_version=4.5.1
+td_agent_version=4.5.2
 
 case $distribution in
   amazon)

diff --git a/fluent-package/yum/systemd-test/install-newly.sh b/fluent-package/yum/systemd-test/install-newly.sh
@@ -7,7 +7,7 @@ set -exu
 case $1 in
   local)
     sudo $DNF install -y \
-      /vagrant/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
+      /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
     ;;
   v5)
     case $DISTRIBUTION in
@@ -46,18 +46,8 @@ test -e /var/log/fluent/fluentd.log
 sudo $DNF remove -y fluent-package
 sudo systemctl daemon-reload
 
-case $1 in
-  local)
-    getent passwd fluentd >/dev/null
-    getent group fluentd >/dev/null
-    ;;
-  *)
-    # TODO: Remove this branch after the following fix is applied to the latest release.
-    # https://github.com/fluent/fluent-package-builder/pull/598
-    (! getent passwd fluentd >/dev/null)
-    (! getent group fluentd >/dev/null)
-    ;;
-esac
+getent passwd fluentd >/dev/null
+getent group fluentd >/dev/null
 # `sudo systemctl daemon-reload` clears the service completely.
 #   (The result of `systemctl status` will be `unfound`)
 # Note: RPM does not leave links like `@/etc/systemd/system/fluentd.service`.

diff --git a/fluent-package/yum/systemd-test/test.sh b/fluent-package/yum/systemd-test/test.sh
@@ -1,46 +1,48 @@
 #!/bin/bash
 
 if [ -z $1 ]; then
-    echo "Error: Need to specify VM name in the Vagrantfile."
-    echo "Ex.) $ ./test.sh almalinux-9"
+    echo "Error: Need to specify distribution name."
+    echo "Ex.) $ ./test.sh centos-7"
     exit 1
 fi
 
-vm=$1
-dir="/vagrant/fluent-package/yum/systemd-test"
+image=$1
+dir="/host/fluent-package/yum/systemd-test"
 
-vagrant status $vm | grep -E "^${vm}\s+not created (.*)$"
-if [ $? -ne 0 ]; then
-    echo "Error: The VM already exists. Need to destroy it in advance with the following command."
-    echo "$ vagrant destroy $vm"
-    exit 1
-fi
-
-set -eu
+set -eux
 
 test_filenames=(
-    update-from-v4.sh
     update-to-next-version.sh
-    update-to-next-version-with-backward-compat-for-v4.sh
 )
 
+if [ ! $image = "images:amazonlinux/2023" ]; then
+    # As no AmazonLinux/2023 package for v4, so execute upgrade test for other images.
+    test_filenames+=(
+        update-from-v4.sh
+        update-to-next-version-with-backward-compat-for-v4.sh
+    )
+fi
+
 for yum_repo_type in local v5 lts; do
     echo -e "\nRun test: $yum_repo_type\n"
-    vagrant up $vm
-    vagrant ssh $vm -- $dir/install-newly.sh $yum_repo_type
-    vagrant destroy -f $vm
+    lxc launch $image target
+    sleep 5
+    lxc config device add target host disk source=$PWD path=/host
+    lxc list
+    lxc exec target -- $dir/install-newly.sh $yum_repo_type
+    lxc stop target
+    lxc delete target
 done
 
 for test_filename in ${test_filenames[@]}; do
     echo -e "\nRun test: $test_filename\n"
-    vagrant up $vm
-    vagrant ssh $vm -- $dir/$test_filename
-    vagrant destroy -f $vm
-    # I want to use snapshot instead of destroying it for every test,
-    # but somehow, it will be often an error on GitHub Actions...
-    #   $ vagrant snapshot save -f $vm after-setup
-    #   (execute a test)
-    #   $ vagrant snapshot restore $vm after-setup
+    lxc launch $image target
+    sleep 5
+    lxc config device add target host disk source=$PWD path=/host
+    lxc list
+    lxc exec target -- $dir/$test_filename
+    lxc stop target
+    lxc delete target
 done
 
 echo -e "\nAll Success!\n"
diff --git a/fluent-package/yum/systemd-test/update-from-v4.sh b/fluent-package/yum/systemd-test/update-from-v4.sh
@@ -36,7 +36,7 @@ systemctl status --no-pager td-agent
 
 # Install the current
 sudo $DNF install -y \
-    /vagrant/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
+    /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
 
 # Test: service status
 systemctl status --no-pager fluentd # Migration process starts the service automatically

diff --git a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh
@@ -35,7 +35,7 @@ sudo systemctl enable --now td-agent
 systemctl status --no-pager td-agent
 
 # Install the current
-package="/vagrant/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm"
+package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm"
 sudo $DNF install -y $package
 systemctl status --no-pager fluentd # Migration process starts the service automatically
 sudo systemctl enable --now fluentd