ci: Add tft plan and workflow

This change is for running tests in Testing Farm CI. This is a replacement for BaseOS CI that we are currently using. Running it Testing Farm gives us more control. It adds a workflow for running tests, and a plans directory containing a test plan and a README-plans.md with some info. Note that this workflow runs from the main branch. This means that changes to the workflow must be merged to main, then pull requests will be able to run it. This is because the workflow uses on: issue_comment context, this is a security measure recommended by GitHub. It saves us from leaking organization secrets. The functionality is WIP, so await future fixes and updates. Signed-off-by: Sergei Petrosian <spetrosi@redhat.com>
linux-system-roles · Jul 24, 2024 · 8d6e840 · 8d6e840
1 parent e8ffa61
commit 8d6e840
Show file tree

Hide file tree

Showing 6 changed files with 272 additions and 53 deletions.
diff --git a/.fmf/version b/.fmf/version
@@ -0,0 +1 @@
+1
diff --git a/.github/workflows/tft.yml b/.github/workflows/tft.yml
@@ -0,0 +1,172 @@
+name: Testing Farm
+on:
+  issue_comment:
+    types:
+      - created
+permissions:
+  contents: read
+  # This is required for the ability to create/update the Pull request status
+  statuses: write
+# The concurrency key is used to prevent multiple workflows from running at the same time
+concurrency:
+  # group name contains reponame-pr_num to allow simualteneous runs in different PRs
+  group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+jobs:
+  prepare_vars:
+    name: Get supported platforms from meta/main.yml
+    # Let's schedule tests only on user request. NOT automatically.
+    # Only repository owner or member can schedule tests
+    if: |
+      github.event.issue.pull_request
+      && (contains(github.event.comment.body, '[citest]') || contains(github.event.comment.body, '[citest-all]'))
+      && (contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || contains('systemroller', github.event.comment.user.login))
+    runs-on: ubuntu-latest
+    outputs:
+      supported_platforms: ${{ steps.supported_platforms.outputs.supported_platforms }}
+      head_sha: ${{ steps.head_sha.outputs.head_sha }}
+      datetime: ${{ steps.datetime.outputs.datetime }}
+      memory: ${{ steps.memory.outputs.memory }}
+    steps:
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Get head sha of the PR
+        id: head_sha
+        run: |
+          head_sha=$(gh api "repos/$REPO/pulls/$PR_NO" --jq '.head.sha')
+          echo "head_sha=$head_sha" >> $GITHUB_OUTPUT
+        env:
+          REPO: ${{ github.repository }}
+          PR_NO: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get cuurent datetime
+        id: datetime
+        run: |
+          printf -v datetime '%(%Y%m%d-%H%M%S)T' -1
+          echo "datetime=$datetime" >> $GITHUB_OUTPUT
+
+      - name: Get memory
+        id: memory
+        run: |
+          memory=$(grep -rPo '    m: \K(.*)' tests/provision.fmf)
+          if [ -n "$memory" ]; then
+            echo "memory=$memory" >> $GITHUB_OUTPUT
+          else
+            echo "memory=2048" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Get supported platforms
+        id: supported_platforms
+        run: |
+          supported_platforms=""
+          meta_main=meta/main.yml
+          # All Fedora are supported, add latest Fedora versions to supported_platforms
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi fedora$; then
+            supported_platforms+=" Fedora-39"
+            supported_platforms+=" Fedora-40"
+          fi
+          # Specific Fedora versions supported
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qiP 'fedora\d+$'; then
+            for fedora_ver in $(yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -iPo 'fedora\K(\d+$)'); do
+              supported_platforms+=" Fedora-$fedora_ver"
+            done
+          fi
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el7; then
+            supported_platforms+=" CentOS-7-latest"
+          fi
+          for ver in 8 9 10; do
+            if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el"$ver"; then
+              supported_platforms+=" CentOS-Stream-$ver"
+            fi
+          done
+          echo "supported_platforms=$supported_platforms" >> $GITHUB_OUTPUT
+
+  testing-farm:
+    name: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
+    needs: prepare_vars
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: Fedora-39
+            ansible_version: 2.17
+          - platform: Fedora-40
+            ansible_version: 2.17
+          - platform: CentOS-7-latest
+            ansible_version: 2.9
+          - platform: CentOS-Stream-8
+            ansible_version: 2.9
+          # On CentOS-Stream-8, latest supported Ansible is 2.16
+          - platform: CentOS-Stream-8
+            ansible_version: 2.16
+          - platform: CentOS-Stream-9
+            ansible_version: 2.17
+          - platform: CentOS-Stream-10
+            ansible_version: 2.17
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACTS_DIR_NAME: "tf_${{ github.event.repository.name }}-${{ github.event.issue.number }}_\
+        ${{ matrix.platform }}-${{ matrix.ansible_version }}_\
+        ${{ needs.prepare_vars.outputs.datetime }}/artifacts"
+      ARTIFACT_TARGET_DIR: /srv/pub/alt/linuxsystemroles/logs
+    steps:
+      - name: Set commit status as pending
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: pending
+          context: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
+          description: Test started
+
+      - name: Set commit status as success with a description that platform is skipped
+        if: "!contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)"
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: success
+          context: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
+          description: The role does not support this platform. Skipping.
+          targetUrl: ""
+
+      - uses: sclorg/testing-farm-as-github-action@v2
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_DIR: ${{ env.ARTIFACT_TARGET_DIR }}/${{ env.ARTIFACTS_DIR_NAME }}
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          git_url: ${{ github.server_url }}/${{ github.repository }}
+          git_ref: ${{ needs.prepare_vars.outputs.head_sha }}
+          pipeline_settings: '{ "type": "tmt-multihost" }'
+          variables: "ANSIBLE_VER=${{ matrix.ansible_version }};\
+            REPO_NAME=${{ github.event.repository.name }};\
+            GITHUB_ORG=linux-system-roles;\
+            PR_NUM=${{ github.event.issue.number }};\
+            ARTIFACTS_DIR=${{ env.ARTIFACTS_DIR }};\
+            ARTIFACTS_URL=${{ env.ARTIFACTS_URL }}"
+          # Note that LINUXSYSTEMROLES_SSH_KEY must be single-line, TF doesn't read multi-line variables fine.
+          secrets: "LINUXSYSTEMROLES_USER=${{ secrets.LINUXSYSTEMROLES_USER }};\
+            LINUXSYSTEMROLES_DOMAIN=${{ secrets.LINUXSYSTEMROLES_DOMAIN }};\
+            LINUXSYSTEMROLES_SSH_KEY=${{ secrets.LINUXSYSTEMROLES_SSH_KEY }}"
+          compose: ${{ matrix.platform }}
+          # There are two blockers for using public ranch:
+          # 1. multihost is not supported in public https://github.com/teemtee/tmt/issues/2620
+          # 2. Security issue that leaks long secrets - Jira TFT-2698
+          tf_scope: private
+          api_key: ${{ secrets.TF_API_KEY_RH }}
+          update_pull_request_status: false
+          tmt_hardware: '{ "memory": ">= ${{ needs.prepare_vars.outputs.memory }} MB" }'
+
+      - name: Set final commit status
+        uses: myrotvorets/set-commit-status-action@master
+        if: always() && contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: ${{ job.status }}
+          targetUrl: ${{ env.ARTIFACTS_URL }}
+          context: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Role Name
 
-[![ansible-lint.yml](https://github.com/linux-system-roles/template/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/template/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/template/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/markdownlint.yml) [![shellcheck.yml](https://github.com/linux-system-roles/template/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/shellcheck.yml) [![woke.yml](https://github.com/linux-system-roles/template/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/woke.yml)
+[![ansible-lint.yml](https://github.com/linux-system-roles/template/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/template/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/template/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/markdownlint.yml) [![shellcheck.yml](https://github.com/linux-system-roles/template/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/template/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/tft.yml) [![woke.yml](https://github.com/linux-system-roles/template/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/template/actions/workflows/woke.yml)
 
 ![template](https://github.com/linux-system-roles/template/workflows/tox/badge.svg)
 

diff --git a/meta/main.yml b/meta/main.yml
@@ -1,71 +1,20 @@
 # SPDX-License-Identifier: MIT
 ---
 galaxy_info:
-  # Replace with role's author name:
   author: John Doe <jdoe@corp.com>
-  # Replace with the real description of what is role's purpose:
   description: Basic template for Linux system roles
-  # Replace with the company the role's author is member of:
   company: John Doe, Inc.
-
-  # If the issue tracker for your role is not on github, uncomment the next
-  # line and provide a value
-  # issue_tracker_url: http://example.com/issue/tracker
-
-  # Some suggested licenses:
-  # - BSD (default)
-  # - MIT
-  # - GPLv2
-  # - GPLv3
-  # - Apache
-  # - CC-BY
   license: MIT
-
   min_ansible_version: "2.9"
-
-  # Optionally specify the branch Galaxy will use when accessing the GitHub
-  # repo for this role. During role install, if no tags are available, Galaxy
-  # will use this branch. During import Galaxy will access files on this
-  # branch. If Travis integration is configured, only notifications for this
-  # branch will be accepted. Otherwise, in all cases, the repo's default branch
-  # (usually main) will be used.
-  # github_branch:
-
-  #
-  # platforms is a list of platforms, and each platform has a name and a list
-  # of versions.
-  #
-  # platforms:
-  # - name: Fedora
-  #   versions:
-  #   - all
-  #   - "25"
-  # - name: SomePlatform
-  #   versions:
-  #   - all
-  #   - "1.0"
-  #   - "7"
-  #   - "99.99"
   platforms:
-    # Replace the below with your platform list:
     - name: Fedora
       versions:
         - all
     - name: EL
       versions:
         - "9"
-
   galaxy_tags:
-    - fedora
     - el9
     - el10
-  # List tags for your role here, one per line. A tag is a keyword that
-  # describes and categorizes the role. Users find roles by searching for tags.
-  # tags are also used to list platform/version support.
-  #
-  # NOTE: A tag is limited to a single word comprised of alphanumeric
-  #       characters. Maximum 20 tags per role.
-
+    - fedora
 dependencies: []
-# List your role dependencies here, one per line. Be sure to remove the '[]'
-# above, if you add dependencies to this list.
diff --git a/plans/README-plans.md b/plans/README-plans.md
@@ -0,0 +1,35 @@
+# Introduction CI Testing Plans
+
+Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tmt.yml](https://github.com/linux-system-roles/template/blob/main/.github/workflows/tmt.yml) GitHub workflow.
+
+The plans/general.fmf plan is a test plan that is general for all roles. It does the following steps:
+
+1. Provisions two machines, one used as an Ansible control node, and second used as a managed node.
+2. Does the required preparation on machines.
+3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh).
+
+The [tmt.yml](https://github.com/linux-system-roles/template/blob/main/.github/workflows/tmt.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access.
+This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm).
+
+## Running Tests
+
+You can run tests locally with the `tmt try` cli.
+
+### Prerequisites
+
+* Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html).
+
+### Running Tests Locally
+
+For now, this functionality requires you to push changes to a PR because the plan only runs from the main branch, or from the PR branch.
+So this is WIP.
+
+To run tests locally, in the role repository, enter `tmt run plans --name plans/general <platform>`.
+Where `<platform>` is the name of the platform you want to run tests against.
+
+For example, `tmt run plans --name plans/general Fedora-40`.
+
+This command identifies the plans/general plan and provisions two machines, one used as an Ansible control node, and second used as a managed node.
+
+You can also use `tmt try` to get to an interreactive prompt and be able to ssh into test machines.
+You must run `tmt try -p plans/general Fedora-40`, and the in the promt type `p` to prepare the machines, then `t` to run the tests.
diff --git a/plans/general.fmf b/plans/general.fmf
@@ -0,0 +1,62 @@
+summary: A general test for a system role
+provision:
+  - name: control_node
+    role: control_node
+    # TF uses `how: artemis`, tmt try uses `how: virtual`. No need to define `how`
+    # `connection: system` is for `how: virtual` to make VMs get a real IP to configure ssh easily
+    # This setting is ignored on artemis so we can keep it
+    connection: system
+  - name: managed_node1
+    role: managed_node
+    connection: system
+environment:
+    ANSIBLE_VER: 2.17
+    REPO_NAME: template
+    PYTHON_VERSION: 3.12
+    SYSTEM_ROLES_ONLY_TESTS: ""
+    PR_NUM: ""
+prepare:
+  - name: Use vault.centos.org repos (CS 7, 8 EOL workaround)
+    script: |
+      if grep -q -e 'CentOS Stream release 8' -e 'CentOS Linux release 7.9' /etc/redhat-release; then
+        sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo
+      fi
+
+  - name: Enable epel to install beakerlib on all platforms except CS10 and Fedora, there it's not available and not needed
+    script: |
+      if ! grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then
+        yum install epel-release -y
+      fi
+    where: control_node
+
+  - name: Additional steps to enable EPEL on EL 7
+    script: |
+      if grep -q 'CentOS Linux release 7.9' /etc/redhat-release; then
+        yum install yum-utils -y
+        yum-config-manager --enable epel epel-debuginfo epel-source
+      fi
+    where: control_node
+
+  - name: Install python on managed node when running CS8 with ansible!=2.9
+    script: |
+      if [ "$ANSIBLE_VER" != "2.9" ] && grep -q 'CentOS Stream release 8' /etc/redhat-release; then
+        dnf install -y python"$PYTHON_VERSION"
+      fi
+    where: managed_node
+
+  - name: Distribute SSH keys when provisioned with how=virtual
+    script: |
+      if [ -f ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub ]; then
+        cat ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub >> ~/.ssh/authorized_keys
+      fi
+    where: managed_node
+
+discover:
+  - name: Run test playbooks from control_node
+    how: fmf
+    url: https://github.com/linux-system-roles/tft-tests
+    ref: main
+    where: control_node
+    filter: tag:test_playbooks
+execute:
+    how: tmt