ci tft tests

.fmf/version

@@ -0,0 +1 @@
+1

.github/workflows/tft.yml

@@ -0,0 +1,177 @@
+name: Testing Farm
+on:
+  issue_comment:
+    types:
+      - created
+permissions:
+  contents: read
+  # This is required for the ability to create/update the Pull request status
+  statuses: write
+# The concurrency key is used to prevent multiple workflows from running at the same time
+concurrency:
+  # group name contains reponame-pr_num to allow simualteneous runs in different PRs
+  group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+jobs:
+  prepare_vars:
+    name: Get supported platforms from meta/main.yml
+    # Let's schedule tests only on user request. NOT automatically.
+    # Only repository owner or member can schedule tests
+    if: |
+      github.event.issue.pull_request
+      && (contains(github.event.comment.body, '[citest]') || contains(github.event.comment.body, '[citest-all]'))
+      && (contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.comment.author_association)
+      || contains('systemroller', github.event.comment.user.login))
+    runs-on: ubuntu-latest
+    outputs:
+      supported_platforms: ${{ steps.supported_platforms.outputs.supported_platforms }}
+      head_sha: ${{ steps.head_sha.outputs.head_sha }}
+      datetime: ${{ steps.datetime.outputs.datetime }}
+      memory: ${{ steps.memory.outputs.memory }}
+    steps:
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Get head sha of the PR
+        id: head_sha
+        run: |
+          head_sha=$(gh api "repos/$REPO/pulls/$PR_NO" --jq '.head.sha')
+          echo "head_sha=$head_sha" >> $GITHUB_OUTPUT
+        env:
+          REPO: ${{ github.repository }}
+          PR_NO: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get cuurent datetime
+        id: datetime
+        run: |
+          printf -v datetime '%(%Y%m%d-%H%M%S)T' -1
+          echo "datetime=$datetime" >> $GITHUB_OUTPUT
+
+      - name: Get memory
+        id: memory
+        run: |
+          if [ -d tests/provision.fmf ]; then
+            memory=$(grep -rPo '    m: \K(.*)' tests/provision.fmf)
+          fi
+          if [ -n "$memory" ]; then
+            echo "memory=$memory" >> $GITHUB_OUTPUT
+          else
+            echo "memory=2048" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Get supported platforms
+        id: supported_platforms
+        run: |
+          supported_platforms=""
+          meta_main=meta/main.yml
+          # All Fedora are supported, add latest Fedora versions to supported_platforms
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi fedora$; then
+            supported_platforms+=" Fedora-39"
+            supported_platforms+=" Fedora-40"
+          fi
+          # Specific Fedora versions supported
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qiP 'fedora\d+$'; then
+            for fedora_ver in $(yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -iPo 'fedora\K(\d+$)'); do
+              supported_platforms+=" Fedora-$fedora_ver"
+            done
+          fi
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el7; then
+            supported_platforms+=" CentOS-7-latest"
+          fi
+          for ver in 8 9 10; do
+            if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el"$ver"; then
+              supported_platforms+=" CentOS-Stream-$ver"
+            fi
+          done
+          echo "supported_platforms=$supported_platforms" >> $GITHUB_OUTPUT
+
+  testing-farm:
+    name: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
+    needs: prepare_vars
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: Fedora-39
+            ansible_version: 2.17
+          - platform: Fedora-40
+            ansible_version: 2.17
+          - platform: CentOS-7-latest
+            ansible_version: 2.9
+          - platform: CentOS-Stream-8
+            ansible_version: 2.9
+          # On CentOS-Stream-8, latest supported Ansible is 2.16
+          - platform: CentOS-Stream-8
+            ansible_version: 2.16
+          - platform: CentOS-Stream-9
+            ansible_version: 2.17
+          - platform: CentOS-Stream-10
+            ansible_version: 2.17
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACTS_DIR_NAME: "tf_${{ github.event.repository.name }}-${{ github.event.issue.number }}_\
+        ${{ matrix.platform }}-${{ matrix.ansible_version }}_\
+        ${{ needs.prepare_vars.outputs.datetime }}/artifacts"
+      ARTIFACT_TARGET_DIR: /srv/pub/alt/linuxsystemroles/logs
+    steps:
+      - name: Set commit status as pending
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: pending
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: Test started
+          targetUrl: ""
+
+      - name: Set commit status as success with a description that platform is skipped
+        if: "!contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)"
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: success
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: The role does not support this platform. Skipping.
+          targetUrl: ""
+
+      - uses: sclorg/testing-farm-as-github-action@v2
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_DIR: ${{ env.ARTIFACT_TARGET_DIR }}/${{ env.ARTIFACTS_DIR_NAME }}
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          git_url: ${{ github.server_url }}/${{ github.repository }}
+          git_ref: ${{ needs.prepare_vars.outputs.head_sha }}
+          pipeline_settings: '{ "type": "tmt-multihost" }'
+          variables: "ANSIBLE_VER=${{ matrix.ansible_version }};\
+            REPO_NAME=${{ github.event.repository.name }};\
+            GITHUB_ORG=linux-system-roles;\
+            PR_NUM=${{ github.event.issue.number }};\
+            ARTIFACTS_DIR=${{ env.ARTIFACTS_DIR }};\
+            ARTIFACTS_URL=${{ env.ARTIFACTS_URL }}"
+          # Note that LINUXSYSTEMROLES_SSH_KEY must be single-line, TF doesn't read multi-line variables fine.
+          secrets: "LINUXSYSTEMROLES_USER=${{ secrets.LINUXSYSTEMROLES_USER }};\
+            LINUXSYSTEMROLES_DOMAIN=${{ secrets.LINUXSYSTEMROLES_DOMAIN }};\
+            LINUXSYSTEMROLES_SSH_KEY=${{ secrets.LINUXSYSTEMROLES_SSH_KEY }}"
+          compose: ${{ matrix.platform }}
+          # There are two blockers for using public ranch:
+          # 1. multihost is not supported in public https://github.com/teemtee/tmt/issues/2620
+          # 2. Security issue that leaks long secrets - Jira TFT-2698
+          tf_scope: private
+          api_key: ${{ secrets.TF_API_KEY_RH }}
+          update_pull_request_status: false
+          tmt_hardware: '{ "memory": ">= ${{ needs.prepare_vars.outputs.memory }} MB" }'
+
+      - name: Set final commit status
+        uses: myrotvorets/set-commit-status-action@master
+        if: always() && contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: ${{ job.status }}
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: Test finished
+          targetUrl: ${{ env.ARTIFACTS_URL }}

README.md

@@ -1,6 +1,6 @@
 # postfix
 
-[![ansible-lint.yml](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/postfix/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/markdownlint.yml) [![woke.yml](https://github.com/linux-system-roles/postfix/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/woke.yml)
+[![ansible-lint.yml](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/postfix/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/markdownlint.yml) [![tft.yml](https://github.com/linux-system-roles/postfix/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/tft.yml) [![woke.yml](https://github.com/linux-system-roles/postfix/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/postfix/actions/workflows/woke.yml)
 
 This role can install, configure and start Postfix MTA.
 

plans/README-plans.md

@@ -0,0 +1,35 @@
+# Introduction CI Testing Plans
+
+Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tmt.yml](https://github.com/linux-system-roles/postfix/blob/main/.github/workflows/tmt.yml) GitHub workflow.
+
+The plans/general.fmf plan is a test plan that is general for all roles. It does the following steps:
+
+1. Provisions two machines, one used as an Ansible control node, and second used as a managed node.
+2. Does the required preparation on machines.
+3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh).
+
+The [tmt.yml](https://github.com/linux-system-roles/postfix/blob/main/.github/workflows/tmt.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access.
+This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm).
+
+## Running Tests
+
+You can run tests locally with the `tmt try` cli.
+
+### Prerequisites
+
+* Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html).
+
+### Running Tests Locally
+
+For now, this functionality requires you to push changes to a PR because the plan only runs from the main branch, or from the PR branch.
+So this is WIP.
+
+To run tests locally, in the role repository, enter `tmt run plans --name plans/general <platform>`.
+Where `<platform>` is the name of the platform you want to run tests against.
+
+For example, `tmt run plans --name plans/general Fedora-40`.
+
+This command identifies the plans/general plan and provisions two machines, one used as an Ansible control node, and second used as a managed node.
+
+You can also use `tmt try` to get to an interreactive prompt and be able to ssh into test machines.
+You must run `tmt try -p plans/general Fedora-40`, and the in the promt type `p` to prepare the machines, then `t` to run the tests.

plans/general.fmf

@@ -0,0 +1,62 @@
+summary: A general test for a system role
+provision:
+  - name: control_node
+    role: control_node
+    # TF uses `how: artemis`, tmt try uses `how: virtual`. No need to define `how`
+    # `connection: system` is for `how: virtual` to make VMs get a real IP to configure ssh easily
+    # This setting is ignored on artemis so we can keep it
+    connection: system
+  - name: managed_node1
+    role: managed_node
+    connection: system
+environment:
+    ANSIBLE_VER: 2.17
+    REPO_NAME: postfix
+    PYTHON_VERSION: 3.12
+    SYSTEM_ROLES_ONLY_TESTS: ""
+    PR_NUM: ""
+prepare:
+  - name: Use vault.centos.org repos (CS 7, 8 EOL workaround)
+    script: |
+      if grep -q -e 'CentOS Stream release 8' -e 'CentOS Linux release 7.9' /etc/redhat-release; then
+        sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo
+      fi
+
+  - name: Enable epel to install beakerlib on all platforms except CS10 and Fedora, there it's not available and not needed
+    script: |
+      if ! grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then
+        yum install epel-release -y
+      fi
+    where: control_node
+
+  - name: Additional steps to enable EPEL on EL 7
+    script: |
+      if grep -q 'CentOS Linux release 7.9' /etc/redhat-release; then
+        yum install yum-utils -y
+        yum-config-manager --enable epel epel-debuginfo epel-source
+      fi
+    where: control_node
+
+  - name: Install python on managed node when running CS8 with ansible!=2.9
+    script: |
+      if [ "$ANSIBLE_VER" != "2.9" ] && grep -q 'CentOS Stream release 8' /etc/redhat-release; then
+        dnf install -y python"$PYTHON_VERSION"
+      fi
+    where: managed_node
+
+  - name: Distribute SSH keys when provisioned with how=virtual
+    script: |
+      if [ -f ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub ]; then
+        cat ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub >> ~/.ssh/authorized_keys
+      fi
+    where: managed_node
+
+discover:
+  - name: Run test playbooks from control_node
+    how: fmf
+    url: https://github.com/linux-system-roles/tft-tests
+    ref: main
+    where: control_node
+    filter: tag:test_playbooks
+execute:
+    how: tmt