chore: Simplify tox, bump dependencies

[holmanb]

Additional Context

• bump dependency versions (black, mypy, etc)
• fix mypy issues for newer versions
• since python3.7 is no longer supported by pytest, make a separate tox environment and CI job for the old version
• switch to Ubuntu 22.04 where needed for improved pip dependency resolution

Merge type

• Squash merge using "Proposed Commit Message"
• Rebase and merge unique commits. Requires commit messages per-commit each referencing the pull request number (#<PR_NUM>)

[TheRealFalcon]

I left some inline comments. I'm assuming that in the black commit, all of the changes there were auto-formatted? Is there anything there that you had to manually update?

[holmanb]

I left some inline comments. I'm assuming that in the black commit, all of the changes there were auto-formatted?

Correct

Is there anything there that you had to manually update?

This commit was manual changes by me for mypy, but otherwise no.

[TheRealFalcon]

Thanks for all the changes! I'm +1 to the PR once the pytest configuration moves back to tox.ini

[TheRealFalcon]

LGTM assuming current checks pass

[holmanb]

Thanks for all the changes! I'm +1 to the PR once the pytest configuration moves back to tox.ini

The review is much appreciated @TheRealFalcon! I just pushed a change which did that, and also reverted an unnecessary test addition to integration tests and fixed linkcheck which I had broken. I squashed changes before merging, so this is the git diff origin/holmanb/simplify-tox just before I force pushed.

diff --git a/pyproject.toml b/pyproject.toml
index 7eadd316a..2adba3761 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -275,27 +275,3 @@ lint.ignore = [
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"
-
-[tool.pytest.ini_options]
-testpaths = "tools tests/unittests"
-addopts = "-vvvv --strict-markers --showlocals --durations 10"
-log_format = "%(asctime)s %(levelname)-9s %(name)s:%(filename)s:%(lineno)d %(message)s"
-log_date_format = "%Y-%m-%d %H:%M:%S"
-markers = [
-    "adhoc: only run on adhoc basis, not in any CI environment (travis or jenkins)",
-    "allow_all_subp: allow all subp usage (disable_subp_usage)",
-    "allow_subp_for: allow subp usage for the given commands (disable_subp_usage)",
-    "ci: run this integration test as part of CI test runs",
-    "ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures",
-    "hypothesis_slow: hypothesis test too slow to run as unit test",
-    "instance_name: the name to be used for the test instance",
-    "integration_cloud_args: args for IntegrationCloud customization",
-    "is_iscsi: whether is an instance has iscsi net cfg or not",
-    "lxd_config_dict: set the config_dict passed on LXD instance creation",
-    "lxd_setup: specify callable to be called between init and start",
-    "lxd_use_exec: `execute` will use `lxc exec` instead of SSH",
-    "serial: tests that do not work in parallel, skipped with py3-fast",
-    "unstable: skip this test because it is flakey",
-    "user_data: the user data to be passed to the test instance",
-    "allow_dns_lookup: disable autochecking for host network configuration",
-]
diff --git a/tox.ini b/tox.ini
index bef415c4f..be5e1d647 100644
--- a/tox.ini
+++ b/tox.ini
@@ -197,7 +197,7 @@ commands = {envpython} -m sphinx -b spelling {posargs:-W doc/rtd doc/rtd_html}
 #
 # followed by manual verification of the links reported
 [testenv:linkcheck]
-deps = {[pinned_versions]deps}
+deps = -r{toxinidir}/doc-requirements.txt
 commands =
     {envpython} -m sphinx {posargs:-b linkcheck doc/rtd doc/rtd_html}
 
@@ -225,9 +225,7 @@ deps = {[latest_versions]deps}
 commands = {envpython} -m isort --check-only --diff {posargs:.}
 
 [testenv:integration-tests]
-deps =
-    {[pinned_versions]deps}
-    -r{toxinidir}/integration-requirements.txt
+deps = -r{toxinidir}/integration-requirements.txt
 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
 passenv =
     CLOUD_INIT_*
@@ -236,9 +234,7 @@ passenv =
     OS_*
 
 [testenv:integration-tests-ci]
-deps =
-    {[pinned_versions]deps}
-    -r{toxinidir}/integration-requirements.txt
+deps = -r{toxinidir}/integration-requirements.txt
 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
 passenv =
     CLOUD_INIT_*
@@ -250,9 +246,7 @@ setenv =
 [testenv:integration-tests-jenkins]
 # Pytest's RC=1 means "Tests were collected and run but some of the tests failed".
 # Do not fail in this case, but let Jenkins handle it using the junit report.
-deps =
-    {[pinned_versions]deps}
-    -r{toxinidir}/integration-requirements.txt
+deps = -r{toxinidir}/integration-requirements.txt
 allowlist_externals = sh
 commands = sh -c "{envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests/none} || [ $? -eq 1 ]"
 passenv =
@@ -266,6 +260,30 @@ passenv =
 setenv =
     PYTEST_ADDOPTS="-m not adhoc"
 
+[pytest]
+# TODO: s/--strict/--strict-markers/ once pytest version is high enough
+testpaths = tools tests/unittests
+addopts = --strict
+log_format = %(asctime)s %(levelname)-9s %(name)s:%(filename)s:%(lineno)d %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+markers =
+    adhoc: only run on adhoc basis, not in any CI environment (travis or jenkins)
+    allow_all_subp: allow all subp usage (disable_subp_usage)
+    allow_subp_for: allow subp usage for the given commands (disable_subp_usage)
+    ci: run this integration test as part of CI test runs
+    ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures
+    hypothesis_slow: hypothesis test too slow to run as unit test
+    instance_name: the name to be used for the test instance
+    integration_cloud_args: args for IntegrationCloud customization
+    is_iscsi: whether is an instance has iscsi net cfg or not
+    lxd_config_dict: set the config_dict passed on LXD instance creation
+    lxd_setup: specify callable to be called between init and start
+    lxd_use_exec: `execute` will use `lxc exec` instead of SSH
+    serial: tests that do not work in parallel, skipped with py3-fast
+    unstable: skip this test because it is flakey
+    user_data: the user data to be passed to the test instance
+    allow_dns_lookup: disable autochecking for host network configuration
+
 [coverage:paths]
 source =
     cloudinit/