[EDITOR-527] E2E testing infrastructure (#591)

* Bootstrap E2E testing infrastructure

* add e2e tests in taskfile and in CI

* fix deprecation warning when running pytest

* Run e2e tests only with GUI version on macos (Github runners are basically mac mini with GUI support)"

* add first draft of tests for certs and v2 APIs

* add logs to .gitignore

* skip update, currently no way of testing this on gh runners

Co-authored-by: umbynos <umberto.baldi@edu.unito.it>

Author: Maurizio Branca

.github/workflows/release.yml

@@ -123,6 +123,19 @@ jobs:
         run: task build
         if: matrix.os == 'macos-10.15'
 
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+        if: matrix.os == 'macos-10.15'
+
+      - name: Run e2e tests
+        run: |
+          pip install poetry
+          task test-e2e
+        if: matrix.os == 'macos-10.15'
+
         # this will create `public/` dir with compressed full bin (<version>/<os>-<arch>.gz) and a json file
       - name: Create autoupdate files
         run: go-selfupdate arduino-create-agent${{ matrix.ext }} ${TAG_VERSION}

.github/workflows/test.yml

@@ -91,6 +91,19 @@ jobs:
         run: task build
         if: matrix.os == 'macos-10.15'
 
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+        if: matrix.os == 'macos-10.15'
+
+      - name: Run e2e tests
+        run: |
+          pip install poetry
+          task test-e2e
+        if: matrix.os == 'macos-10.15'
+
         # config.ini is required by the executable when it's run
       - name: Upload artifacts
         uses: actions/upload-artifact@v2

.gitignore

@@ -7,9 +7,14 @@ rsrc.syso
 snapshot/*
 public/
 artifacts*
+logs/
 
 # IDEs config 
 .idea
+.vscode
 
 # macOS
 .DS_Store
+
+# Python
+__pycache__

Taskfile.yml

@@ -33,6 +33,12 @@ tasks:
     cmds:
       - go test -short -run '{{ default ".*" .TEST_REGEX }}' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}}
 
+  test-e2e:
+    desc: Run end 2 end tests
+    cmds:
+      - poetry install --no-root
+      - poetry run pytest test
+
   check:
     desc: Check fmt and lint
     cmds:

poetry.lock

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "arduino-create-agent"
+version = "0.1.0"
+description = "Project used to run end-to-end test for the Arduino Create Agent"
+authors = ["Umberto Baldi <u.baldi@arduino.cc>", "Maurizio Branca <m.branca@arduino.cc>"]
+license = "GPLv2"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+psutil = "^5.8.0"
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.1"
+requests = "^2.25.1"
+invoke = "^1.5.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore::DeprecationWarning:invoke.loader" # https://github.com/pyinvoke/invoke/issues/675
+]