Force series16 to install dependencies (#661)

* Force snapcraft.yaml for s16 to install deps

There does not seem to be another way to install the dependencies
using the pyproject key. This is due to the fact that there is no
pip/setuptools/other backend that seem to support it on such an old
version of python

* Force also metabox source runs to install deps

Minor: python may not be python3, call python3

* New test plan check installation

* Dependency test script for metabox-provider

* Add metabox scenario to validate installation

* Renamed all jobs to the same name

* Removed test plan

* Tag test

* Minor: removed empty line

checkbox-core-snap/series16/snap/snapcraft.yaml

@@ -173,12 +173,21 @@ parts:
  override-build: |
  # this is a bodge to be backward compatible
  snapcraftctl build
- # also use build to ensure install (pip is not compinat to pyproject)
- # on this version
+ # setup.py is necessary for this old pip verision
  echo "from setuptools import setup; setup()" > setup.py
+ # update pip and setuptools (+deps) because the directive
+ # is ignored in pyproject (and pip on xenial is preinstalled 8.x)
  $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install "pip<21" "setuptools<48" "setuptools_scm[toml]>=3.4" "importlib_metadata==1.0.0" "zipp<2"
- # also, call pip again, this installs the module itself
+ # install the dependencies of checkbox-support itself because this
+ # pip version pyproject.toml dependencies
+ $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install "pyparsing<3.0.0" "requests<2.26.0" "distro<1.7.0" "requests_unixsocket<=0.3.0" "importlib_metadata<=1.0.0"
+ # call pip again to actually install the module itself
  $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install .
+ # fix shebangs in bins.This is necessary because by calling pip via the
+ # $SNAPCRAFT_PART_INSTALL path (to get the correct one) this path gets
+ # propagated into bins. This path will be invalid once the snap is installed
+ # so it must be changed to the appropriate one (/usr/bin/env python3) that
+ # we could not use here
  sed -iE 's|\#\!.*/usr/bin/python3$|#!/usr/bin/env python3|' $SNAPCRAFT_PART_INSTALL/usr/local/bin/*
 ################################################################################
  checkbox-ng:
@@ -197,6 +206,7 @@ parts:
  - python3-urwid
  - python3-xlsxwriter
  - python3-setuptools
+ - python3-dev
  python-packages:
  - tqdm
  after: [checkbox-support]
@@ -206,20 +216,22 @@ parts:
  override-build: |
  # this is a bodge to be backward compatible
  snapcraftctl build
- # also use build to ensure install (pip is not compinat to pyproject)
- # on this version
+ # setup.py is necessary for this old pip verision
  echo "from setuptools import setup; setup()" > setup.py
- # setup the correct pip and related tools version
+ # update pip and setuptools (+deps) because the directive
+ # is ignored in pyproject (and pip on xenial is preinstalled 8.x)
  $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install "pip<21" "setuptools<48" "setuptools_scm[toml]>=3.4" "importlib_metadata==1.0.0" "zipp<2"
- # install checkbox-ng
+ # install the dependencies of checkbox-support itself because this
+ # pip version pyproject.toml dependencies
+ $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install "packaging<21.0" "psutil<=5.9.5" "requests<2.26.0" "urwid<=2.1.2" "Jinja2<=2.11.3" "XlsxWriter<=3.0.3" "tqdm<4.65.0" "importlib_metadata<=1.0.0"
+ # call pip again to actually install the module itself
  $SNAPCRAFT_PART_INSTALL/usr/bin/python3 -m pip install .
  # fix shebangs in bins.This is necessary because by calling pip via the
  # $SNAPCRAFT_PART_INSTALL path (to get the correct one) this path gets
  # propagated into bins. This path will be invalid once the snap is installed
  # so it must be changed to the appropriate one (/usr/bin/env python3) that
  # we could not use here
  sed -iE 's|\#\!.*/usr/bin/python3$|#!/usr/bin/env python3|' $SNAPCRAFT_PART_INSTALL/usr/local/bin/*
- #sed -i "1s/.*/#!\/usr\/bin\/env python3/" $SNAPCRAFT_PART_INSTALL/usr/local/bin/*
 ################################################################################
  checkbox-provider-resource:
  plugin: dump

metabox/metabox/core/machine.py

@@ -286,6 +286,17 @@ def _get_install_dependencies_cmds(self):
  # Use <21 to get the latest 20 as 21+ is not supported here
  return [
  "bash -c 'sudo python3 -m pip install -U \"pip<21\"'",
+ (
+ # This is here because this pip version does not support
+ # installing dependencies from pyproject.toml
+ "bash -c 'sudo python3 -m pip install "
+ '"pyparsing<3.0.0" "requests<2.26.0" "distro<1.7.0" '
+ '"requests_unixsocket<=0.3.0" "importlib_metadata<=1.0.0"'
+ '"packaging<21.0" "psutil<=5.9.5" "requests<2.26.0" '
+ '"urwid<=2.1.2" "Jinja2<=2.11.3" "XlsxWriter<=3.0.3" '
+ '"tqdm<4.65.0" "importlib_metadata<=1.0.0"'
+ "'"
+ ),
  ]
  if self.config.alias not in ["focal", "jammy"]:
  logger.warning(
@@ -563,7 +574,7 @@ def get_setup_overlay_fs(self):
  gid=0,
  )
  install_metabox_provider = (
- "sudo /snap/checkbox/current/bin/wrapper_local python "
+ "sudo /snap/checkbox/current/bin/wrapper_local python3 "
  "/home/ubuntu/metabox-provider/manage.py install "
  "--layout=relocatable --prefix=/providers/metabox-provider "
  "--root={snap_runtime_location}"

metabox/metabox/metabox-provider/bin/dependency_installation.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Canonical Ltd.
+#
+# Authors:
+# Massimiliano Girardi <massimiliano.girardi@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+This module imports all dependencies of checkbox-ng and checkbox-support.
+This allows us to verify that all dependencies are actually installed by
+the install process, without this modules checkbox does not work!
+
+If you have updated the dependency list of the following, update this test
+as well:
+- checkbox-ng/pyproject.toml
+- checkbox-support/pyproject.toml
+"""
+
+# Core checkbox module
+import checkbox_ng
+
+# checkbox-ng dependencies
+import jinja2
+import packaging
+import psutil
+import requests
+import tqdm
+import urwid
+import xlsxwriter
+
+try:
+ import importlib_metadata
+except ModuleNotFoundError:
+ import importlib.metadata
+
+# Used by checkbox and providers
+import plainbox
+
+# Contains various help functions/scripts
+import checkbox_support
+
+# checkbox-support dependencies
+import distro
+import pyparsing
+import requests
+import requests
+import requests_unixsocket
+
+try:
+ import importlib_metadata
+except ModuleNotFoundError:
+ import importlib.metadata

metabox/metabox/metabox-provider/units/basic-jobs.pxu

@@ -237,3 +237,9 @@ _description:
  VERIFICATION:
  1. Did the 3d animation appear?
  2. Was the animation free from slowness/jerkiness?
+
+plugin: shell
+id: dependency_installation
+command: dependency_installation.py
+_summary: Verify that all required modules are installed
+_description: Verify that all required modules are installed

metabox/metabox/scenarios/installation/validation.py

@@ -0,0 +1,35 @@
+# This file is part of Checkbox.
+#
+# Copyright 2023 Canonical Ltd.
+# Written by:
+# Massimiliano Girardi <massimiliano.girardi@canonical.com>
+#
+# Checkbox is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+#
+# Checkbox is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.
+
+from metabox.core.actions import AssertPrinted, Start
+from metabox.core.scenario import Scenario
+from metabox.core.utils import tag
+
+
+@tag("dependencies", "installation")
+class DependencyInstallation(Scenario):
+ """
+ This verifies that the checkbox installation installed
+ all that is needed
+ """
+
+ modes = ["local"]
+ steps = [
+ Start("run 2021.com.canonical.certification::dependency_installation"),
+ AssertPrinted("Outcome: job passed"),
+ ]