diff --git a/.gitignore b/.gitignore index 88c8f3585..bf7287630 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /enter.bash *.dtb venv/ +pip-cache/ diff --git a/.gitmodules b/.gitmodules index 99155dd91..49f6a97e7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -130,9 +130,6 @@ [submodule "scripts/cmsis-svd-generator"] path = scripts/cmsis-svd-generator url = https://github.com/sifive/cmsis-svd-generator.git -[submodule "pip-cache"] - path = pip-cache - url = https://github.com/sifive/freedom-e-sdk-pip-cache.git [submodule "scripts/openocdcfg-generator"] path = scripts/openocdcfg-generator url = https://github.com/sifive/openocdcfg-generator.git diff --git a/Makefile b/Makefile index 849544759..fc08163f6 100644 --- a/Makefile +++ b/Makefile @@ -275,7 +275,6 @@ standalone: \ $(STANDALONE_DEST)/bsp \ $(STANDALONE_DEST)/src \ $(SRC_DIR) \ - pip-cache \ freedom-metal \ debug.mk \ release.mk \ @@ -291,8 +290,6 @@ standalone: \ scripts/virtualenv.mk cp -r $(addprefix $(BSP_DIR)/,$(filter-out build,$(shell ls $(BSP_DIR)))) $=30.3.0 wheel setuptools_scm>=3.3.1 -pip +pip==20.0.1 diff --git a/scripts/download-python-packages.py b/scripts/download-python-packages.py new file mode 100755 index 000000000..bef9fcf71 --- /dev/null +++ b/scripts/download-python-packages.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +from itertools import product + +if __name__ == "__main__": + requirements = sys.argv[1] + destination = sys.argv[2] + + platforms = ["manylinux1_x86_64", "macosx_10_13_x86_64", "win32", "win_amd64"] + versions = ["35", "36", "37"] + + tuples = product(platforms, versions) + + base_cmd = ["python3", "-m", "pip", "download", "-d", destination, "-r", requirements] + + subprocess.run(base_cmd + ["--platform", "any", "--no-deps"]) + + for platform, version in tuples: + subprocess.run(base_cmd + ["--platform", platform, "--python-version", version, "--only-binary=:all:"]) + + print(list(tuples)) diff --git a/scripts/virtualenv.mk b/scripts/virtualenv.mk index 6c5bddf8f..51a457fca 100644 --- a/scripts/virtualenv.mk +++ b/scripts/virtualenv.mk @@ -7,42 +7,63 @@ FREEDOM_E_SDK_VENV_PATH ?= venv .PHONY: virtualenv virtualenv: $(FREEDOM_E_SDK_VENV_PATH)/.stamp -# Just in case the cached dependencies fail, users can install Python packages -# from the Python Package Index by running `make virtualenv-from-pypi` - -.PHONY: virtualenv-from-pypi - -# invoke python3 pip in order to bypass 127 character limit in shebang -virtualenv-from-pypi: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt - . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --upgrade pip - . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install -r requirements.txt +.PHONY: clean-virtualenv +clean-virtualenv: + -rm -rf $(FREEDOM_E_SDK_VENV_PATH) -$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt - . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links pip-cache --upgrade pip - . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links pip-cache -r requirements.txt - touch $@ +# Freedom E SDK has two options for fetching Python dependencies: +# +# 1. Dependencies are fetched from the Python Package Index (PyPI) when the +# virtualenv is created. +# 2. Dependencies are prefeteched from PyPI and stored in +# FREEDOM_E_SDK_PIP_CACHE_PATH, then installed from there at virtualenv +# creation time. +# +# By default, the pip-cache is created in the `pip-cache` folder at the root of +# freedom-e-sdk. If you want the pip-cache to be placed elsewhere, set the +# FREEDOM_E_SDK_PIP_CACHE_PATH environment variable. +FREEDOM_E_SDK_PIP_CACHE_PATH ?= pip-cache $(FREEDOM_E_SDK_VENV_PATH)/bin/activate: python3 -m venv $(FREEDOM_E_SDK_VENV_PATH) + . $@ && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install pip==20.0.1 -# The pip-cache directory holds a cache of all the Python package dependencies, being careful -# that all are platform-indepentent. -# -# The result should be that when freedom-e-sdk is cloned, the virtualenv is created simply -# by installing all cached dependencies from the submodule -# -# The pip-cache can be updated when requirements.txt changes by running `make update-pip-cache`, -# commiting the changes to the submodule, and bumping the submodule. +.PHONY: pip-cache +pip-cache: $(FREEDOM_E_SDK_PIP_CACHE_PATH)/.stamp -.PHONY: update-pip-cache -update-pip-cache: requirements.txt - ./pip-cache/download-python-dependencies.py $< +$(FREEDOM_E_SDK_PIP_CACHE_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt + . $< && python3 scripts/download-python-packages.py requirements.txt $(FREEDOM_E_SDK_PIP_CACHE_PATH) + touch $@ .PHONY: clean-pip-cache clean-pip-cache: - -rm -rf pip-cache/*.whl pip-cache *.tar.gz + -rm -r $(FREEDOM_E_SDK_PIP_CACHE_PATH) -.PHONY: clean-virtualenv -clean-virtualenv: - -rm -rf $(FREEDOM_E_SDK_VENV_PATH) +# If FREEDOM_E_SDK_PIP_CACHE_PATH does not exist, fetch dependencies from PyPI +ifeq ("$(wildcard $(FREEDOM_E_SDK_PIP_CACHE_PATH))","") + +# We invoke pip as a script in order to avoid depending on the shebang line, which might +# exceed the 127-character limit depending on your environment. + +$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt + @echo "################################################################################" >&2 + @echo "FREEDOM_E_SDK_PIP_CACHE not found, creating virtualenv from Python Package Index" >&2 + @echo "You can pre-download Python packages by running 'make pip-cache'." >&2 + @echo "################################################################################" >&2 + . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install pip==20.0.1 + . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install -r requirements.txt + touch $@ + +else # If FREEDOM_E_SDK_PIP_CACHE_PATH does exist, fetch dependences from there + +# We invoke pip as a script in order to avoid depending on the shebang line, which might +# exceed the 127-character limit depending on your environment. + +$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt + @echo "##################################################################" >&2 + @echo "FREEDOM_E_SDK_PIP_CACHE found, creating virtualenv from $(FREEDOM_E_SDK_PIP_CACHE_PATH)" >&2 + @echo "##################################################################" >&2 + . $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links $(FREEDOM_E_SDK_PIP_CACHE_PATH) -r requirements.txt + touch $@ +endif # FREEDOM_E_SDK_PIP_CACHE_PATH diff --git a/wit-manifest.json b/wit-manifest.json index 92cfa85b4..b31f5386c 100644 --- a/wit-manifest.json +++ b/wit-manifest.json @@ -14,11 +14,6 @@ "name": "elf2hex", "source": "git@github.com:sifive/elf2hex.git" }, - { - "commit": "e73944211dfc5b5493f6c1e19fe8bd0cd1464257", - "name": "pip-cache", - "source": "git@github.com:sifive/freedom-e-sdk-pip-cache.git" - }, { "commit": "726d8761fba8108d4038c050dbd80d539766a11b", "name": "devicetree-overlay-generator",