wheel.mk: Redesign proposal (#4951)

* wheel.mk: Redesign proposal based on @hgy59 feedbacks
* native/python310: Downgrade crossenv 1.0 to fix missing pip
* wheel.mk: Allow passing prefix to wheel=x.y in WHEELS variable
* python310: Add wheel=x.y to WHEELS variable to showcase
* wheel.mk: Adjust identation
* native/python310: Revert back to crossenv==1.1.4
* cross/wheels*: Update pip and setuptools to latest versions
* native/python310: Revert back to crossenv==1.1.4
* wheel.mk: Allow using requirements-pure|cross.txt files
* wheel.mk: Enfore pure|cross on dedicated files
* python310: Add poetry and remote pure: to use default behavior
* python310: Add regex and netifaces
* pillow: Downgrading to 8.2.0 per @hgy59 request
* beets: Update to python310
* tvheadend: Migrate to python310
* python310: Update requirements using cross as default
* python310: Remove temporary examples in WHEELS definition
* wheel.mk: Fix cross-compiling (based on @publicarray finding)
* python3(legacy): Align pip and setuptools with python310
* octoprint: Update to python310
* sickchill: Update to python310
* tvheadend: Remove python virtualenv as all wheels in python310
* octoprint: Migrate service-setup to use python310
* sickchill: Update service-setup to use python310
* beets: Update service-setup to use python310
* flexget: Migrate to python310
* python3: Adjust requirements.txt
* python38: Adjust requirements
* sabnzbd: Update to python 3.10
* download.mk: Temporary add --no-certificate-check to wget for yasm
* beets: Update native pip path for python310
* duplicity: Migrate to python 3.10
* borgbackup: Migrate to python 3.10
* deluge: Adjust requirements to latest changes
* python310: Add poetry to crossenv - e.g. ONE FIX TO RULE THEM ALL
* python310: Rework requirements with poetry now in crossenv
* sickchill: All wheels are now cross thnx to poetry in crossenv
* python310: Move basic wheels from WHEELS to requirements.txt
* octoprint: Migrate pure to cross wheels using poetry
* sabnzbd: Migrate additional pure to cross wheels due to poetry
* flexget: Migrate pure to cross requirements due to poetry
* rdiff-backup: Migrate to python310
* python3: Align with findings from python310
* python38: Align with findings from python310
* mercurial: Migrate to python310
* ffsync: Merge back into a single requirement.txt file
* beets: Fix build issue with python-mdp2
* tvheadend: Revert changes to stop building ffmpeg (ref: #4965)
* python310: Allow building wheels with limited API (abi3 prefix)
* python310: pycrypto is no longer maintained
* wheel.mk: Manage arch specific file prefixes
* pillow: Re-updating to version 8.4.0 per @hgy59 request
* download.mk: Remove temporary no-certificate-check
* wheel.mk: Fix naming for ARMv5 88f6281 as armv5tel
* wheel.mk: Only support requirement files in WHEELS variable
* borgbackup: Use cross & pure requirement files
* duplicity: Use cross & pure requirement files
* flexget: Use cross & pure requirement files
* octoprint: requirement.txt entirely managed through cross-compiled
* python38: Use cross & pure requirement files
* python3: Use cross & pure requirement files
* sabnzbd: Use cross & pure requirement files
* python310: Remove forgotten pure wheels from cross requirement
* sickchill: Bump package version for python 3.10 migration
* python310: Remove cryptography from abi3 (and keep in cross)
* homeassistant: Rough 1st pass to migrate to python310
* Update cython & numpy to latest version to fix x86_64 builds
* python310: Add pycares
* homeassistant: Misc cleanups
* homeassistant: Now fully functional with Python 3.10
* sickchill: Fix typo
* sabnzbd: pytz wheel already provided by python310
* sickchill: BUILD_DEPENDS to python310
* octoprint: Small adjustments to get clean install using python310
* flexget: Disable python310 default wheels
* beets: Now compatible with python310 with update to 1.5.0
* borgbackup: Misc fixes for python310 compatibility
* duplicity: Misc fixes for python310 compabitility
* duplicity: Disable future wheel as already in python310
* znc: Migrate to Python 3.10
* python-wheel.mk: Rename using PYTHON_ARCH instead of none-any.whl
* Rename requirements-cross.txt to requirements-crossenv.txt

cross/cython/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = cython
-PKG_VERS = 0.29.21
+PKG_VERS = 0.29.24
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://github.com/$(PKG_NAME)/$(PKG_NAME)/archive

cross/cython/digests

@@ -1,3 +1,3 @@
-cython-0.29.21.tar.gz SHA1 718dcfee83132421e2df4eb6fa0ef7f3917bf7b9
-cython-0.29.21.tar.gz SHA256 e2e38e1f0572ca54d6085df3dec8b607d20e81515fb80215aed19c81e8fe2079
-cython-0.29.21.tar.gz MD5 e14ffbc4019f9bbb37d977dac46f7d0d
+cython-0.29.24.tar.gz SHA1 36a2a5fe9730758cebf3de28c0ff7ae33595700f
+cython-0.29.24.tar.gz SHA256 a5efb97612f0f97164e87c54cc295b2e2d06c539487670079963adeab872de80
+cython-0.29.24.tar.gz MD5 6a7bd0c2d7a9f1630d82e957197ea6e8

cross/dtlssocket/PLIST

@@ -1 +1 @@
-rsc:bin/wheel
+

cross/numpy/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = numpy
-PKG_VERS = 1.21.1
+PKG_VERS = 1.21.4
 PKG_EXT = zip
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/n/$(PKG_NAME)

cross/numpy/digests

@@ -1,3 +1,3 @@
-numpy-1.21.1.zip SHA1 09b07eb84f565ab67d1d0d1c45ec47f74dbe2a1c
-numpy-1.21.1.zip SHA256 dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd
-numpy-1.21.1.zip MD5 1d016e05851a4ba85307f3246eb569aa
+numpy-1.21.4.zip SHA1 91a1faeb617601453ebf6e2855e5ce6bf94d3588
+numpy-1.21.4.zip SHA256 e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0
+numpy-1.21.4.zip MD5 b3c4477a027d5b6fba5e1065064fd076

cross/pillow/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = Pillow
-PKG_VERS = 8.4.0
+PKG_VERS = 8.2.0
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/P/$(PKG_NAME)

cross/pillow/digests

@@ -1,3 +1,3 @@
-Pillow-8.4.0.tar.gz SHA1 ca8d057aec64ccd0804044bc71c17921afbe2e4e
-Pillow-8.4.0.tar.gz SHA256 b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed
-Pillow-8.4.0.tar.gz MD5 7a1eb5a250c7ccbd549a89e16404f09f
+Pillow-8.2.0.tar.gz SHA1 a9b32caf6ff1d5160ad41265c7ad3b090114eefc
+Pillow-8.2.0.tar.gz SHA256 a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1
+Pillow-8.2.0.tar.gz MD5 21c03274a9f59b9c00419852a8faebe7

cross/pip/Makefile

@@ -1,6 +1,6 @@
 PKG_NAME = pip
 # use the same version in native/python3/Makefile (python3_native_post_install)
-PKG_VERS = 21.3
+PKG_VERS = 21.3.1
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/p/$(PKG_NAME)

cross/pip/digests

@@ -1,3 +1,3 @@
-pip-21.3.tar.gz SHA1 e6c8b84c60a290818079ce7ebc71ccbf9cd7d294
-pip-21.3.tar.gz SHA256 741a61baab1dbce2d8ca415effa48a2b6a964564f81a9f4f1fce4c433346c034
-pip-21.3.tar.gz MD5 fc20feba27bd36f046c2fbd59bc8f9ea
+pip-21.3.1.tar.gz SHA1 a243525070cf70f22a185447ebd3e1435dabb218
+pip-21.3.1.tar.gz SHA256 fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a
+pip-21.3.1.tar.gz MD5 0d3f27f4b7fecb33fd573e4f46cc6788

cross/python3/Makefile

@@ -83,18 +83,26 @@ python3_custom_install:
 	@install -m 644 src/mime.types $(STAGING_INSTALL_PREFIX)/etc/
 	$(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) $(MAKE) install prefix=$(STAGING_INSTALL_PREFIX)
 
+# Create the crossenv in preparation for
+# cross-compiling all the necessary wheels
 .PHONY: python3_post_install
 python3_post_install: $(WORK_DIR)/python-cc.mk
 	mkdir -p $(PYTHON_LIB_CROSS)
 	cp -R $(HOSTPYTHON_LIB_NATIVE) $(PYTHON_LIB_CROSS)/../
 	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) $(WORK_DIR)/crossenv/
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) wget https://bootstrap.pypa.io/get-pip.py
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==21.3"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==21.3"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "setuptools==58.2.0"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "setuptools==58.2.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==21.3.1"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==21.3.1"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "setuptools==58.3.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "setuptools==58.3.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "wheel==0.37.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "wheel==0.37.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cffi==1.15.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cffi==1.15.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "poetry==1.1.11"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "poetry==1.1.11"
 ifneq ($(PYTHON_LIB_NATIVE),$(PYTHON_LIB_CROSS))
 	cp $(PYTHON_LIB_CROSS)/_sysconfigdata_*.py $(PYTHON_LIB_NATIVE)/_sysconfigdata.py
 endif

cross/python310/Makefile

@@ -125,6 +125,8 @@ python310_install:
 	@install -m 644 src/mime.types $(STAGING_INSTALL_PREFIX)/etc/
 	$(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) $(MAKE) install prefix=$(STAGING_INSTALL_PREFIX)
 
+# Create the crossenv in preparation for
+# cross-compiling all the necessary wheels
 .PHONY: python310_post_install
 python310_post_install: $(WORK_DIR)/python-cc.mk
 	mkdir -p $(PYTHON_LIB_CROSS)
@@ -139,11 +141,15 @@ python310_post_install: $(WORK_DIR)/python-cc.mk
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "wheel==0.37.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cffi==1.15.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cffi==1.15.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "poetry==1.1.11"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "poetry==1.1.11"
 ifneq ($(PYTHON_LIB_NATIVE),$(PYTHON_LIB_CROSS))
 	cp $(PYTHON_LIB_CROSS)/_sysconfigdata_*.py $(PYTHON_LIB_NATIVE)/_sysconfigdata.py
 endif
 
-$(WORK_DIR)/python-cc.mk:
+$(WORK_DIR)/python-cc.mk: 
 	@echo HOSTPYTHON=$(HOSTPYTHON) > $@
 	@echo CROSSENV=$(WORK_DIR)/crossenv/bin/activate >> $@
 	@echo HOSTPYTHON_LIB_NATIVE=$(HOSTPYTHON_LIB_NATIVE) >> $@

cross/python38/Makefile

@@ -124,20 +124,26 @@ python38_install:
 	@install -m 644 src/mime.types $(STAGING_INSTALL_PREFIX)/etc/
 	$(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) $(MAKE) install prefix=$(STAGING_INSTALL_PREFIX)
 
+# Create the crossenv in preparation for
+# cross-compiling all the necessary wheels
 .PHONY: python38_post_install
 python38_post_install: $(WORK_DIR)/python-cc.mk
 	mkdir -p $(PYTHON_LIB_CROSS)
 	cp -R $(HOSTPYTHON_LIB_NATIVE) $(PYTHON_LIB_CROSS)/../
 	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) $(WORK_DIR)/crossenv/
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) wget https://bootstrap.pypa.io/get-pip.py
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==21.3"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==21.3"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "setuptools==58.2.0"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "setuptools==58.2.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==21.3.1"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==21.3.1"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "setuptools==58.3.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "setuptools==58.3.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "wheel==0.37.0"
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "wheel==0.37.0"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cffi==1.14.6"
-	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cffi==1.14.6"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cffi==1.15.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cffi==1.15.0"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "cryptography==3.3.2"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-pip install "poetry==1.1.11"
+	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) pip install "poetry==1.1.11"
 ifneq ($(PYTHON_LIB_NATIVE),$(PYTHON_LIB_CROSS))
 	cp $(PYTHON_LIB_CROSS)/_sysconfigdata_*.py $(PYTHON_LIB_NATIVE)/_sysconfigdata.py
 endif

cross/setuptools/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = setuptools
-PKG_VERS = 58.2.0
+PKG_VERS = 58.3.0
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/s/$(PKG_NAME)

cross/setuptools/digests

@@ -1,3 +1,3 @@
-setuptools-58.2.0.tar.gz SHA1 5d97de0e774b2269c85685a4aa8fd5956bcfd2bb
-setuptools-58.2.0.tar.gz SHA256 2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145
-setuptools-58.2.0.tar.gz MD5 ff20ab7e0d51c5ad0a9438c50e598c06
+setuptools-58.3.0.tar.gz SHA1 d5354718cb8c9330d3abc27445467ce8a5ed9d70
+setuptools-58.3.0.tar.gz SHA256 b0c2461641b58fe30e11d4c3dfba316c513bdf9ec85f9fed0c871c678447205e
+setuptools-58.3.0.tar.gz MD5 7a2c32ef46b0f91acc8c2756af56a711

cross/znc/Makefile

@@ -5,7 +5,7 @@ PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://znc.in/releases/archive
 PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
 
-BUILD_DEPENDS = cross/python38
+BUILD_DEPENDS = cross/python310
 DEPENDS = cross/openssl cross/libicu
 
 # due to libicu:

mk/spksrc.common.mk

@@ -13,13 +13,19 @@ MSG = echo "===> "
 # Launch command in the working dir of the package source and the right environment
 RUN = cd $(WORK_DIR)/$(PKG_DIR) && env $(ENV)
 
-# Pip command
+# fallback by default to native/python*
 PIP ?= pip
+
+# Cross compiling must call "pip" directly to ensure using
+# $(WORK_DIR)/crossenv pip version instead from adjusted $PATH
+PIP_CROSS = $(shell . $(CROSSENV) && $(RUN) which pip)
+
+# System default pip outside from build environment
+PIP_SYSTEM = $(shell which pip)
+
 # Why ask for the same thing twice? Always cache downloads
 PIP_CACHE_OPT ?= --cache-dir $(PIP_DIR)
 PIP_WHEEL_ARGS = wheel --no-binary :all: $(PIP_CACHE_OPT) --no-deps --wheel-dir $(WHEELHOUSE)
-PIP_WHEEL = $(PIP) $(PIP_WHEEL_ARGS)
-PIP_DOWNLOAD = $(PIP) download $(PIP_ARGS) --dest $(BASE_DISTRIB_DIR)
 
 # Available languages
 LANGUAGES = chs cht csy dan enu fre ger hun ita jpn krn nld nor plk ptb ptg rus spn sve trk

mk/spksrc.python-wheel.mk

@@ -7,10 +7,13 @@ ifeq ($(strip $(CONFIGURE_TARGET)),)
 CONFIGURE_TARGET = nop
 endif
 ifeq ($(strip $(COMPILE_TARGET)),)
-COMPILE_TARGET = build_python_wheel
+COMPILE_TARGET = build_python_wheel_target
 endif
 ifeq ($(strip $(INSTALL_TARGET)),)
-INSTALL_TARGET = install_python_wheel
+INSTALL_TARGET = nop
+endif
+ifeq ($(strip $(POST_INSTALL_TARGET)),)
+POST_INSTALL_TARGET = post_install_python_wheel_target
 endif
 
 # Resume with standard spksrc.cross-cc.mk
@@ -22,24 +25,20 @@ include ../../mk/spksrc.cross-cc.mk
 # Python module variables
 PYTHONPATH = $(PYTHON_LIB_NATIVE):$(INSTALL_DIR)$(INSTALL_PREFIX)/$(PYTHON_LIB_DIR)/site-packages/
 
+## python wheel specific configurations
+include ../../mk/spksrc.wheel-env.mk
 
 ### Python wheel rules
-build_python_wheel:
+build_python_wheel_target:
 ifeq ($(strip $(CROSSENV)),)
 # Python 2 way
 	@$(RUN) PYTHONPATH=$(PYTHONPATH) $(HOSTPYTHON) -c "import setuptools;__file__='setup.py';exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" $(BUILD_ARGS) bdist_wheel -d $(WHEELHOUSE)
 else
 # Python 3 case: using crossenv helper
 	@. $(CROSSENV) && $(RUN) PYTHONPATH=$(PYTHONPATH) python -c "import setuptools;__file__='setup.py';exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" $(BUILD_ARGS) bdist_wheel -d $(WHEELHOUSE)
 endif
+	@$(RUN) echo "$(PKG_NAME)==$(PKG_VERS)" >> $(WHEELHOUSE)/$(WHEELS_CROSS_COMPILE)
 
-install_python_wheel: $(WHEEL_TARGET)
-	@if [ -d "$(WHEELHOUSE)" ] ; then \
-		mkdir -p $(STAGING_INSTALL_WHEELHOUSE) ; \
-		cd $(WHEELHOUSE) && \
-		  for w in *.whl; do \
-		    cp -f $$w $(STAGING_INSTALL_WHEELHOUSE)/`echo $$w | cut -d"-" -f -3`-none-any.whl; \
-		  done ; \
-	fi
+post_install_python_wheel_target: $(WHEEL_TARGET) install_python_wheel
 
 all: install