From af9837712ad34602b099fdb587eb72cfd9d8fff3 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 21:27:41 -0700 Subject: [PATCH 1/9] Add "Additional resources" links to readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 7be3edd..5007a09 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ This action checks whether [Arduino](https://www.arduino.cc/) sketches compile a - [How it works](#how-it-works) - [`enable-warnings-report`](#enable-warnings-report) - [Example usage](#example-usage) +- [Additional resources](#additional-resources) @@ -226,3 +227,9 @@ Set to `true` to cause the action to record the compiler warning count for each - name: Stepper version: 1.1.3 ``` + +## Additional resources + +- [Introductory article about **arduino/compile-sketches**](https://blog.arduino.cc/2021/04/09/test-your-arduino-projects-with-github-actions/) +- [**GitHub Actions** documentation](https://docs.github.com/actions/learn-github-actions/understanding-github-actions) +- [Discuss or request assistance on **Arduino Forum**](https://forum.arduino.cc/) From c4fa302d60bbbaf1c91517a5cd38c3c31a0ffdee Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 21:28:33 -0700 Subject: [PATCH 2/9] Add an FAQ document This will serve as a container for more in-depth documentation to supplement the fundamental reference provided in the readme. --- README.md | 1 + docs/FAQ.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 docs/FAQ.md diff --git a/README.md b/README.md index 5007a09..abae657 100644 --- a/README.md +++ b/README.md @@ -231,5 +231,6 @@ Set to `true` to cause the action to record the compiler warning count for each ## Additional resources - [Introductory article about **arduino/compile-sketches**](https://blog.arduino.cc/2021/04/09/test-your-arduino-projects-with-github-actions/) +- [Frequently asked questions about **arduino/compile-sketches**](docs/FAQ.md#frequently-asked-questions) - [**GitHub Actions** documentation](https://docs.github.com/actions/learn-github-actions/understanding-github-actions) - [Discuss or request assistance on **Arduino Forum**](https://forum.arduino.cc/) diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 0000000..cc37fe9 --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,43 @@ +# Frequently Asked Questions + +## How can I install dependencies of a boards platform? + +### Managed Dependencies + +The Arduino **Boards Manager** system installs tool dependencies along with a platform. When you specify a [**Boards Manager**-sourced platform dependency](../README.md#boards-manager) via the action's [`platforms` input](../README.md#platforms) the managed platform dependencies are installed automatically. + +If an alternative [platform dependency source](../README.md#supported-platform-sources) is used this automatic tool dependency installation does not occur. The convenient way to install the tool dependencies in this case is to install a **Boards Manager**-sourced platform that has a dependency on the required tools in addition to the target platform from the alternative source. + +--- + +**Example:** + +```yaml +- uses: arduino/compile-sketches@v1 + with: + platforms: | + # Use Boards Manager to install the latest release of the platform to get the toolchain. + - name: arduino:avr + # Overwrite the Boards Manager installation with the development version of the platform. + - source-url: https://github.com/arduino/ArduinoCore-avr.git + name: arduino:avr +``` + +--- + +### External Dependencies + +Arduino boards platforms typically bundle all dependencies. However, there are some platforms that require the user to manually install dependencies on their system in order to use the platform. + +The **arduino/compile-sketches** action runs in the same environment as the rest of the steps of the [workflow job](https://docs.github.com/actions/using-jobs/using-jobs-in-a-workflow), which means you can simply perform the dependency installation in a prior [step](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps) of the job. + +--- + +**Example:** + +```yaml +- run: +- uses: arduino/compile-sketches@v1 +``` + +--- From 042dd4a28f263b0608987d15e27507e5fd5d849e Mon Sep 17 00:00:00 2001 From: per1234 Date: Sat, 18 Mar 2023 02:58:53 -0700 Subject: [PATCH 3/9] Test support of external Python package platform dependencies Although Arduino boards platforms traditionally have bundled all dependencies, there are some cases where the user will need to use a preceeding workflow step to install external dependencies in the GitHub Actions runner environment. Since it is a "composite action", the action supports this. The action uses a virtual environment for its own Python package dependencies. This virtual environment might cause Python packages installed via the user's workflow (e.g., the pyserial dependency of the ESP32 platform) to not be accessible to the compilation commands. The added integration test will catch any regressions in the support for external Python package platform dependencies. --- .github/workflows/test-integration.yml | 26 +++++++++++++++++++ .../PythonPackageDependent/boards.txt | 8 ++++++ .../PythonPackageDependent/dependent.py | 2 ++ .../PythonPackageDependent/platform.txt | 3 +++ 4 files changed, 39 insertions(+) create mode 100644 .github/workflows/testdata/platforms/PythonPackageDependent/boards.txt create mode 100644 .github/workflows/testdata/platforms/PythonPackageDependent/dependent.py create mode 100644 .github/workflows/testdata/platforms/PythonPackageDependent/platform.txt diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index b40a9eb..f4aa46d 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -19,6 +19,7 @@ on: env: SKETCHES_REPORTS_PATH: sketches-reports + TESTDATA_PLATFORMS_PATH: .github/workflows/testdata/platforms TESTDATA_SKETCHES_PATH: .github/workflows/testdata/sketches TESTDATA_REPORTS_PATH: .github/workflows/testdata/reports @@ -203,6 +204,31 @@ jobs: sketch-paths: | - examples/Sweep + python-package-dependency: + runs-on: ubuntu-latest + + steps: + - name: Checkout local repo + uses: actions/checkout@v3 + + - name: Install Python package dependency + run: pip install cowsay + + - name: Run action with board that has external Python package dependency + # Use action from local path + uses: ./ + with: + platforms: | + - name: arduino:avr + - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PythonPackageDependent + name: PythonPackageDependent:avr + fqbn: PythonPackageDependent:avr:package_dependent + libraries: | + [] + sketch-paths: | + - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum + + check-sketches-reports: needs: all-inputs runs-on: ubuntu-latest diff --git a/.github/workflows/testdata/platforms/PythonPackageDependent/boards.txt b/.github/workflows/testdata/platforms/PythonPackageDependent/boards.txt new file mode 100644 index 0000000..41554a0 --- /dev/null +++ b/.github/workflows/testdata/platforms/PythonPackageDependent/boards.txt @@ -0,0 +1,8 @@ +package_dependent.name=External Python Package Dependent Board +package_dependent.upload.maximum_size=32256 +package_dependent.upload.maximum_data_size=2048 +package_dependent.build.mcu=atmega328p +package_dependent.build.f_cpu=16000000L +package_dependent.build.board=FOO +package_dependent.build.core=arduino:arduino +package_dependent.build.variant=arduino:standard diff --git a/.github/workflows/testdata/platforms/PythonPackageDependent/dependent.py b/.github/workflows/testdata/platforms/PythonPackageDependent/dependent.py new file mode 100644 index 0000000..b372e82 --- /dev/null +++ b/.github/workflows/testdata/platforms/PythonPackageDependent/dependent.py @@ -0,0 +1,2 @@ +# Import external package to verify Python package dependencies can be installed by user into runner environment. +import cowsay # noqa: F401 diff --git a/.github/workflows/testdata/platforms/PythonPackageDependent/platform.txt b/.github/workflows/testdata/platforms/PythonPackageDependent/platform.txt new file mode 100644 index 0000000..ada9a86 --- /dev/null +++ b/.github/workflows/testdata/platforms/PythonPackageDependent/platform.txt @@ -0,0 +1,3 @@ +name=External Python Package Dependent Platform +version=0.0.0 +recipe.hooks.prebuild.1.pattern=python "{runtime.platform.path}/dependent.py" From 4c2850340f7be5ab270d6506df5f9f1d1f8c4c86 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 01:24:17 -0700 Subject: [PATCH 4/9] Restore support for external platform Python package dependencies Although Arduino boards platforms traditionally have bundled all dependencies, there are some cases where the user will need to use a preceding workflow step to install external dependencies in the GitHub Actions runner environment. Since it is a "composite action", this is feasible and was previously supported. However, support for Python package dependencies is more complicated due to the action script running in a Python virtual environment which isolates it from packages installed globally. The workaround for this was to use the `--system-site-packages` flag in the venv invocation that created the action's Python virtual environment. This approach introduced a requirement that the project Python version match the minor version series of the runner system Python. That requirement was violated when the workflow's runner was updated to `ubuntu-22.04` which has system Python 3.10.x, whereas the action used 3.8. The breakage went unnoticed because the sole Python package platform dependency in real world usage, pyserial, had been preinstalled in the runner's Linux package manager "site-packages" folder, which caused it to be installed in the action's virtual environment even though it would not have if depending on the user installation of the package as was previously the case. The solution is to use a path configuration file to add the path of the system Python's "user site-packages" folder (where package dependencies installed via the workflow are located) to the module search paths used by the action's Python virtual environment. This is actually superior to the previous approach of using the `--system-site-packages` flag in the venv invocation (this flag is also supported by pipx and Poetry) because: - It avoids the chance of interference with the virtual environment through the unnecessary introduction of the global "site-packages" path into the module search paths along with the intended introduction of the "user site-packages" path. - It allows any version of Python to be used to run the action, rather than being forced to use the same minor version series as the runner's system Python (which can be updated at any time and also depends on the workflow configuration) --- .github/workflows/test-integration.yml | 6 +++++- action.yml | 24 ++++++++++++++++++++++++ docs/FAQ.md | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index f4aa46d..d5af059 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -212,7 +212,11 @@ jobs: uses: actions/checkout@v3 - name: Install Python package dependency - run: pip install cowsay + run: | + pip install \ + --ignore-installed \ + --user \ + cowsay - name: Run action with board that has external Python package dependency # Use action from local path diff --git a/action.yml b/action.yml index 6c450f1..29cd16b 100644 --- a/action.yml +++ b/action.yml @@ -49,6 +49,13 @@ inputs: runs: using: composite steps: + # User installations of external Python package platform dependencies will be located here. + - name: Get system Python "user site-packages" path + id: system-user-site-packages + shell: bash + run: | + echo "path=$(python -m site --user-site)" >> $GITHUB_OUTPUT + - name: Install Python uses: actions/setup-python@v4.5.0 with: @@ -67,6 +74,23 @@ runs: poetry install \ --only main + - name: Make user-installed Python packages available to platforms + shell: bash + working-directory: ${{ github.action_path }} + run: | + readonly PYTHON_ENVIRONMENT_PATH="$( + poetry env info \ + --path + )" + readonly VENV_SITE_PACKAGES_PATH="$( + poetry run \ + python -c \ + 'import site; print(site.getsitepackages()[0])' + )" + echo \ + "${{ steps.system-user-site-packages.outputs.path }}" > \ + "${VENV_SITE_PACKAGES_PATH}/system-user-site-packages.pth" + - name: Run script shell: bash env: diff --git a/docs/FAQ.md b/docs/FAQ.md index cc37fe9..ae56065 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -41,3 +41,20 @@ The **arduino/compile-sketches** action runs in the same environment as the rest ``` --- + +#### Python Packages + +The **arduino/compile-sketches** action uses a Python [virtual environment](https://docs.python.org/glossary.html#term-virtual-environment). In order to enable user installation of Python [package](https://docs.python.org/glossary.html#term-package) dependencies of boards platforms, the packages installed in the "[user site-packages](https://peps.python.org/pep-0370/)" folder are included in this virtual environment. + +In order to be certain your installation of a package dependency will be available to the platform, add the [`--ignore-installed`](https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-ignore-installed) and [`--user`](https://pip.pypa.io/en/stable/cli/pip_install/#install-user) flags to the [**pip**](https://pip.pypa.io/) command used to install the package. + +--- + +**Example:** + +```yaml +- run: pip install --ignore-installed --user pyserial +- uses: arduino/compile-sketches@v1 +``` + +--- \ No newline at end of file From 32b69515e6c2c744ee9359f15d17a8b1a0afd756 Mon Sep 17 00:00:00 2001 From: per1234 Date: Mon, 20 Mar 2023 01:13:56 -0700 Subject: [PATCH 5/9] Remove superfluous absolute path to action script It is no longer necessary to specify the full path to the script in the invocation command now that the steps working directory has been set to the root folder of the action repository. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 29cd16b..d32a065 100644 --- a/action.yml +++ b/action.yml @@ -108,4 +108,4 @@ runs: working-directory: ${{ github.action_path }} run: | poetry run \ - python "${{ github.action_path }}/compilesketches/compilesketches.py" + python compilesketches/compilesketches.py From add981214aae776e5177610e093a1fe39fa502a4 Mon Sep 17 00:00:00 2001 From: per1234 Date: Mon, 20 Mar 2023 19:14:32 -0700 Subject: [PATCH 6/9] Remove leftover items in workflow path filter Previously, a script was used to install Python and venv for use by the action. The path of this script was added to the paths filter in the relevant workflows so that they would be triggered on any change to the script. The script was removed in favor of the use of the actions/setup-python script for installation of Python. The script path was not removed from the workflow at that time. --- .github/workflows/test-integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index d5af059..46929cd 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -6,7 +6,6 @@ on: - '.github/workflows/test-integration.yml' - '.github/workflows/testdata/**' - 'action.yml' - - 'action-setup.sh' - 'compilesketches/**' push: @@ -14,7 +13,6 @@ on: - '.github/workflows/test-integration.yml' - '.github/workflows/testdata/**' - 'action.yml' - - 'action-setup.sh' - 'compilesketches/**' env: From dc45fe17606c6609e3aefe9d1ee93571a9f3437c Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 17:13:47 -0700 Subject: [PATCH 7/9] Use project Python to install Poetry By default, pipx uses the runner's system Python installation to install Poetry. Although this works, it is best to use the controlled Python installation (installed via the `actions/setup-python` step) for all project operations. --- action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yml b/action.yml index d32a065..06fd288 100644 --- a/action.yml +++ b/action.yml @@ -65,6 +65,7 @@ runs: shell: bash run: | pipx install \ + --python "$(which python)" \ poetry==1.4.0 - name: Install Python Dependencies From 26f371d11d74630214bd5be4a41295cd5355679b Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 21:57:38 -0700 Subject: [PATCH 8/9] Test support of pyserial external dependency of ESP32 platform Although Arduino boards platforms traditionally have bundled all dependencies, the ESP32 boards platform expects the pyserial Python package to be preinstalled on Linux systems. Although there is already test coverage and support for user installation of arbitrary Python package dependencies of a platform, the situation is more complex when it comes to this important boards platform. The problem is that there is an established use pattern that should be supported to avoid breaking user workflows. The added test checks for regressions in that support. --- .github/workflows/test-integration.yml | 29 +++++++++++++++++++ .../platforms/PyserialDependent/boards.txt | 8 +++++ .../platforms/PyserialDependent/platform.txt | 5 ++++ 3 files changed, 42 insertions(+) create mode 100644 .github/workflows/testdata/platforms/PyserialDependent/boards.txt create mode 100644 .github/workflows/testdata/platforms/PyserialDependent/platform.txt diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 46929cd..73b2607 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -231,6 +231,35 @@ jobs: - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum + # Targeted testing for ESP32 boards platform support. + pyserial-dependency: + runs-on: ubuntu-latest + + steps: + - name: Checkout local repo + uses: actions/checkout@v3 + + - name: Install pyserial + run: | + # Use of pip3 and omission of recommended flags done to reproduce established use pattern: + # https://github.com/arduino-libraries/ArduinoIoTCloud/blob/1.11.0/.github/workflows/compile-examples.yml#L206 + pip3 install pyserial + + - name: Run action with board that has pyserial dependency + # Use action from local path + uses: ./ + with: + platforms: | + - name: arduino:avr + - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PyserialDependent + name: PyserialDependent:avr + fqbn: PyserialDependent:avr:pyserial_dependent + libraries: | + [] + sketch-paths: | + - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum + + check-sketches-reports: needs: all-inputs runs-on: ubuntu-latest diff --git a/.github/workflows/testdata/platforms/PyserialDependent/boards.txt b/.github/workflows/testdata/platforms/PyserialDependent/boards.txt new file mode 100644 index 0000000..ba4d86b --- /dev/null +++ b/.github/workflows/testdata/platforms/PyserialDependent/boards.txt @@ -0,0 +1,8 @@ +pyserial_dependent.name=pyserial Dependent Board +pyserial_dependent.upload.maximum_size=32256 +pyserial_dependent.upload.maximum_data_size=2048 +pyserial_dependent.build.mcu=atmega328p +pyserial_dependent.build.f_cpu=16000000L +pyserial_dependent.build.board=FOO +pyserial_dependent.build.core=arduino:arduino +pyserial_dependent.build.variant=arduino:standard diff --git a/.github/workflows/testdata/platforms/PyserialDependent/platform.txt b/.github/workflows/testdata/platforms/PyserialDependent/platform.txt new file mode 100644 index 0000000..11879ca --- /dev/null +++ b/.github/workflows/testdata/platforms/PyserialDependent/platform.txt @@ -0,0 +1,5 @@ +name=pyserial Dependent Platform +version=0.0.0 +# python3 is used to parallel ESP32 platform configuration: +# https://github.com/espressif/arduino-esp32/blob/2.0.7/platform.txt#L220 +recipe.hooks.prebuild.1.pattern=python3 -c 'import serial' From 69fd7b0d9e3a195bbf1c23b50fb7530455a085b8 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 22 Mar 2023 23:33:55 -0700 Subject: [PATCH 9/9] Add pyserial package to action virtual environment Although Arduino boards platforms traditionally have bundled all dependencies, the ESP32 boards platform expects the pyserial Python package to be preinstalled on Linux systems. Although there is already test coverage and support for user installation of arbitrary Python package dependencies of a platform, the situation is more complex when it comes to this important boards platform. The problem is that there is an established use pattern that should be supported to avoid breaking user workflows. The pattern is the use of a `pip install pyserial` command in the workflow to install the package. That command correctly installs the package in the "user site-packages" folder when the job runs in the `ubuntu-18.04` GitHub Actions runner, as was the case when the use pattern was established. However, pyserial is installed in the Linux package manager "site-packages" folder of the Ubuntu GitHub Actions runner starting from ubuntu-20.04 (most workflows run the arduino/compile-sketches job in the `ubuntu-latest` runner, which is currently `ubuntu-22.04`). This causes the established installation command to terminate without installing the package: ``` Requirement already satisfied: pyserial in /usr/lib/python3/dist-packages (3.5) ``` This did not cause a problem previously because the action was configured to include all system site-packages in the action's Python virtual environment. Because this approach resulted in the uncontrolled introduction of irrelevant packages into the action's Python virtual environment, the action was changed to only include the "user site-packages" in the virtual environment. That is fine if the now-recommended `--ignore-installed` flag is added to the pyserial installation command, but it breaks the established workflows. Although the goal was always to provide a flexible system that would allow the user installation of any arbitrary platform dependency, rather than hardcoding specific dependencies in the action itself, unfortunately I have not been able to find any way to progress in the improvement of the action without breaking the established workflows that compile for ESP32 boards. So as a workaround I am reluctantly adding the pyserial package to the action's virtual environment. This workaround will be reverted before the 2.0.0 release of the action so the pyserial installation command should be retained in workflows and occurrences of the historical `pip install pyserial` (or `pip3 install pyserial`) commands should be updated to the recommended command: ``` pip install --ignore-installed --user pyserial ``` --- action.yml | 2 +- poetry.lock | 19 +++++++++++++++++-- pyproject.toml | 8 ++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 06fd288..cb1e415 100644 --- a/action.yml +++ b/action.yml @@ -73,7 +73,7 @@ runs: working-directory: ${{ github.action_path }} run: | poetry install \ - --only main + --only main,external - name: Make user-installed Python packages available to platforms shell: bash diff --git a/poetry.lock b/poetry.lock index cf33fcb..0f7fd8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. [[package]] name = "attrs" @@ -515,6 +515,21 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +[[package]] +name = "pyserial" +version = "3.5" +description = "Python Serial Port Extension" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pyserial-3.5-py2.py3-none-any.whl", hash = "sha256:c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0"}, + {file = "pyserial-3.5.tar.gz", hash = "sha256:3c77e014170dfffbd816e6ffc205e9842efb10be9f58ec16d3e8675b4925cddb"}, +] + +[package.extras] +cp2110 = ["hidapi"] + [[package]] name = "pytest" version = "7.2.2" @@ -756,4 +771,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "3.11.2" -content-hash = "f10d0924625935c307af749c2ade20a628383337648854a7665165563b3aba49" +content-hash = "35ddf7146c03b75108eb3c7396e518ef8066fcfd2c1094eed4f560c68a5242a2" diff --git a/pyproject.toml b/pyproject.toml index 2e27768..a3f0901 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,14 @@ pytest-mock = "3.10.0" flake8 = "5.0.4" pep8-naming = "0.13.3" +# Provided only for use by boards platforms +# NOTE: This group is a temporary workaround that will be removed at the 2.0.0 release of the action. +[tool.poetry.group.external] +optional = true + +[tool.poetry.group.external.dependencies] +pyserial = "3.5" + [tool.pytest.ini_options] filterwarnings = [ "error",