0.8.5 cleanups, infrastructure improvements, ready for a new driver

.circleci/config.yml

@@ -24,15 +24,15 @@ jobs:
           command: |
             sudo pip3 install -e .[dev,test,alldns]
 
-      # run tests!
+      # run tests!  Most config is in pyprojects.toml rather than --options since 0.8.4
       - run:
           name: run tests
           command: |
             make testdata
             find . -type f -name *.pyc -delete | echo
             coverage erase
-            # coverage run --omit="*tests*,*.virtualenvs/*,*.venv/*,*__init__*,*/usr/local/lib/python2.7/dist-packages*" -m unittest discover && bash <(curl -s https://codecov.io/bash)
-            coverage run && ln -s tests/coverage .coverage && bash <(curl -s https://codecov.io/bash)
+            coverage run && ln -s tests/coverage/.coverage .coverage && bash <(curl -s https://codecov.io/bash)
+
       - run:
           name: run tests reports
           command: |      

.gitignore

@@ -28,9 +28,9 @@ README.html
 .vscode/
 .DS_Store
 
-# Coverage artifacts
-/tests/coverage
-/tests/htmlcov/
+# testing artifacts, temp files, etc.
+/tests/coverage/
+/tests/tmp/
 
 # Setuptools distribution folder.
 /dist/

Makefile

@@ -11,71 +11,67 @@ pip = ${python} -m pip
 coverage = ${python} -m coverage
 black = ${python} -m black
 pylint = ${python} -m pylint
+mypy = ${python} -m mypy
 
 
 VERSION_STRING=$$(sed -n -e '/"version"/ s/.*version": *"\([^"]*\)".*/\1/p' <sewer/meta.json)
 
 
-# foo is just a show-me target
-foo:
-	@echo "VERSION = ${VERSION_STRING}"
-	@echo "python is ${python}"
-	@echo "pip command: ${pip}"
-	@echo "twine command: ${twine}"
-	@echo "coverage command: ${coverage}"
-	@echo "black command: ${black}"
-	@echo "pylint command: ${pylint}"
+.PHONY: build
+build:				# build distribution artifacts
+	rm -rf build
+	rm -rf dist
+	rm -rf sewer.egg-info
+	${python} setup.py sdist
+	${python} setup.py bdist_wheel
 
 
-# upload to testpypi
-upload: build
+uploadtest: build		# build and upload to pypi-test
 	@${twine} upload dist/* -r testpypi
 	@${pip} install -U -i https://testpypi.python.org/pypi sewer
 
-build:
-	@rm -rf build
-	@rm -rf dist
-	@rm -rf sewer.egg-info
-	@${python} setup.py sdist
-	@${python} setup.py bdist_wheel
+release2pypi: build upload2pypi release-tag	# build & upload to pypi
+	@echo "${pip} install -U sewer"
 
-uploadprod: build uploadprod_only uploadprod_tag
-	@${pip} install -U sewer
+.PHONY: upload2pypi release-tag
+upload2pypi:
+	${twine} upload dist/*
 
-uploadprod_only:
-	@${twine} upload dist/*
-
-uploadprod_tag:
+release-tag:
 	@printf "\n creating git tag: $(VERSION_STRING) \n"
 	@printf "\n with commit message, see Changelong: https://github.com/komuw/sewer/blob/master/CHANGELOG.md \n" && git tag -a "$(VERSION_STRING)" -m "see Changelong: https://github.com/komuw/sewer/blob/master/CHANGELOG.md"
 	@printf "\n git push the tag::\n" && git push --all -u --follow-tags
 
 
-# you can run single testcase as;
-# 1. python -m unittest sewer.tests.test_Client.TestClient.test_something
-# 2. python -m unittest discover -k test_find_dns_zone_id
+# TESTS - target "test" runs the unit tests under coverage and reports both.
 
 TDATA = tests/data
 
-.PHONY: clean coverage format lint
+.PHONY: clean coverage format-check lint mypy
 
-test: testdata coverage format lint
+test: testdata coverage mypy lint format-check
 
 testdata: rsatestkeys secptestkeys
+	-mkdir tests/tmp
 
 coverage: clean
 	@printf "\n coverage erase::\n" && ${coverage} erase
 	@printf "\n coverage run::\n" && ${coverage} run 
 	@printf "\n coverage report::\n" && ${coverage} report --show-missing --fail-under=85
 
 clean:
-	@printf "\n removing pyc files::\n" && find . -type f -name *.pyc -delete | echo
+	find . -type f -name *.pyc -delete | echo
+	-rm -r tests/tmp
+	-mkdir tests/tmp
 
-format:
-	@printf "\n run black::\n" && ${black} .
+mypy:
+	${mypy} sewer/client.py sewer/cli.py
 
 lint:
-	@printf "\n run pylint::\n" && ${pylint} --enable=E --disable=W,R,C --unsafe-load-any-extension=y sewer/
+	@printf "\n run pylint::\n" && ${pylint} --enable=E --disable=W,R,C --unsafe-load-any-extension=y ${LINTARGS} sewer/
+
+format-check:
+	@printf "\n run black::\n" && ${black} --check .
 
 rsatestkeys: ${TDATA}/rsa2048.pem ${TDATA}/rsa3072.pem ${TDATA}/rsa4096.pem
 

README.md

@@ -8,15 +8,17 @@
 Sewer is a Let's Encrypt(ACME) client.  
 It's name is derived from Kenyan hip hop artiste, Kitu Sewer.  
 
-- The stable release is [0.8.4](https://komuw.github.io/sewer/notes/0.8.4-notes).
-- More history in the [CHANGELOG](https://komuw.github.io/sewer/CHANGELOG).
-
-PYTHON compatibility: 3.5 is nominally still supported, and with assistance
-from Github's multi-version Python linting I've repaired some issues.  Even
-with the GH multi-version testing, there are parts of the drivers,
-especially, which I simply can't test (with 3.5) because the actual service
-provider interaction has to be mocked.  Such issues will be fixed on a
-best-effort basis when reported for the life of 0.8.
+- The stable release is
+  [0.8.4](https://komuw.github.io/sewer/notes/0.8.4-notes).
+- More history (including notes on 0.8.5-to-be) in the
+  [CHANGELOG](https://komuw.github.io/sewer/CHANGELOG).
+
+PYTHON compatibility: 3.5 is still supported, though I believe at least one
+driver contains incompatible code (I have no way to actually test it, and
+sometimes these new features have been backported in 3.x.later).  As of
+0.8.5 I'm still trying not to break 3.5, but that oldest officially
+supported version is probably going to take a bump.  You already cannot run
+all the dev tests under 3.5 (black is one obvious one)...
 
 I (maintainer @mmaney) loiter in channel ##sewer (on irc.freenode.net) for
 those who remember IRC.  Don't ask to ask, but waiting is.

docs/Aliasing.md

@@ -39,7 +39,7 @@ often needed since the privilege needed to directly configure the canonical
 response file is likely to be the same (or even less) than that needed to
 setup the new certificate.  And it's possible that you've already used it
 without thinking of it as _aliasing_ because it uses such basic HTTP
-behavior.
+behavior (and so needs no support from sewer).
 
 ## Preparing for DNS aliasing
 
@@ -86,6 +86,4 @@ redirects can be provisioned.
 This is really pretty short & sweet.
 All that's needed, once the setup is done, is to pass `alias=alias.org` to
 the alias-supporting driver when it's created.
-For users of the command line tool, you would add an option `--alias_domain
-alias.org` as well as specifying a DNS driver that supports the alias
-method, when those become available.
+From the command line, that's `--p_opts alias=alias.org`.

docs/CHANGELOG.md

@@ -1,5 +1,36 @@
 # `sewer` changelog:
 
+## **pre-release** 0.8.5
+
+- driver for Windows DNS server (local only) [IN PROGRESS]
+
+- cleanup that was deferred from 0.8.4 (affects developers, not cli users)
+
+  - crypto.py refactored
+
+  - mypy added to tests
+
+    - dns_providers have had non-base imports cleaned up: use local `# type:
+      ignore` annotations
+
+    - a few non-service-specific libs marked globally to be ignored
+
+  - REMOVED obsolescent dns_provider_name class variables (use the JSON
+    catalog, added in 0.8.3)
+
+  - REMOVED obsolescent guards around service-specific imports and the
+    corresponding delayed exceptions (the unnecessary imports that used to
+    require the guards were removed in 0.8.3)
+
+  - crypto.py's tests migrated to pytest format as tests/crypto_test.py
+
+- Fixed the alias support code and unbound_ssh, its only in-tree client, to
+  use correct names for alias option parameters
+
+- Aliasing document updated to current client options
+
+- in-tree tests began migrating to pytest format (and moving to ./tests)
+
 ## **version:** 0.8.4
 
 - add support for ECDSA keys

mypy.ini

@@ -0,0 +1,15 @@
+# ignoring missing annotations is still a way of life.  In general, add
+# mypy-package here for general-purpose libraries (cryptography.x509.oid and
+# tldextract are good examples); use local "# type: ignore" markup for the
+# service-specific libraries used in drivers (eg. boto3 in route53.py)
+#
+# sadly, Guido resists using pyproject.toml, so this file must add its clutter.
+
+[mypy]
+
+
+[mypy-cryptography.x509.oid]
+    ignore_missing_imports = True
+
+[mypy-tldextract]
+    ignore_missing_imports = True

pyproject.toml

@@ -5,7 +5,7 @@ omit = [
     "*test*",
     "*__init__.py"
 ]
-data_file = "tests/coverage"
+data_file = "tests/coverage/.coverage"
 
 [tool.coverage.report]
 fail_under = 85
@@ -17,12 +17,15 @@ omit = [
 ]
 
 [tool.coverage.html]
-directory = "tests/data/htmlcov"
+directory = "tests/data/coverage/html"
+
 
 [tool.pytest.ini_options]
 console_output_style = "classic"
 addopts = "--color=no"
 
+
+# nigri delenda est - getting to be more trouble than help (no, NOT kwargs, )
 [tool.black]
 line-length = 100
 target-version = [
@@ -31,4 +34,3 @@ target-version = [
     "py37",
     "py38"
 ]
-

setup.py

@@ -51,7 +51,7 @@
     extras_require=dict(
         provider_deps_map,
         dev=["twine", "wheel"],
-        test=["coverage>=5.0", "pytest>=6.0", "pylint==2.3.1", "black==19.10b0"],
+        test=["mypy>=0.780", "coverage>=5.0", "pytest>=6.0", "pylint>=2.6.0", "black==19.10b0"],
         alldns=all_deps_of_all_providers,
     ),
     # data files to be placed in project directory, not zip safe but zips suck anyway

sewer/auth.py

@@ -89,10 +89,10 @@ def __init__(self, *, alias: str = "", **kwargs: Any) -> None:
     def cname_domain(self, chal: Dict[str, str]) -> Union[str, None]:
         "returns fqdn where CNAME should be if aliasing, else None"
 
-        return "_acme-challenge." + chal["domain"] if self.alias else None
+        return "_acme-challenge." + chal["ident_value"] if self.alias else None
 
     def target_domain(self, chal: Dict[str, str]) -> str:
         "returns fqdn where challenge TXT should be placed"
 
-        d = chal["domain"]
+        d = chal["ident_value"]
         return "_acme-challenge." + d if not self.alias else d + "." + self.alias

sewer/client.py

@@ -504,7 +504,7 @@ def get_acme_header(self, url, needs_jwk=False):
         - "url"
         """
         self.logger.debug("get_acme_header")
-        header = {"alg": self.account.key_desc.jwk["alg"], "nonce": self.get_nonce(), "url": url}
+        header = {"alg": self.account.key_desc.alg, "nonce": self.get_nonce(), "url": url}
 
         if needs_jwk:
             header["jwk"] = self.account.jwk()