Merge pull request #447 from sifive/make-pip-cache-optional

Fetch Python packages from PyPI by default, keep pip-cache option

.gitignore

@@ -4,3 +4,4 @@
 /enter.bash
 *.dtb
 venv/
+pip-cache/

.gitmodules

@@ -130,9 +130,6 @@
 [submodule "scripts/cmsis-svd-generator"]
 	path = scripts/cmsis-svd-generator
 	url = https://github.com/sifive/cmsis-svd-generator.git
-[submodule "pip-cache"]
-	path = pip-cache
-	url = https://github.com/sifive/freedom-e-sdk-pip-cache.git
 [submodule "scripts/openocdcfg-generator"]
 	path = scripts/openocdcfg-generator
 	url = https://github.com/sifive/openocdcfg-generator.git

Makefile

@@ -275,7 +275,6 @@ standalone: \
 		$(STANDALONE_DEST)/bsp \
 		$(STANDALONE_DEST)/src \
 		$(SRC_DIR) \
-		pip-cache \
 		freedom-metal \
 		debug.mk \
 		release.mk \
@@ -291,8 +290,6 @@ standalone: \
 		scripts/virtualenv.mk
 	cp -r $(addprefix $(BSP_DIR)/,$(filter-out build,$(shell ls $(BSP_DIR)))) $</bsp/
 
-	cp -r pip-cache $</
-
 	cp -r freedom-metal $</
 	find $</freedom-metal -name ".git*" | xargs rm -rf
 
@@ -368,7 +365,6 @@ standalone: \
 		$(STANDALONE_DEST)/bsp \
 		$(STANDALONE_DEST)/src \
 		$(SRC_DIR) \
-		pip-cache \
 		freedom-metal \
 		debug.mk \
 		release.mk \
@@ -383,8 +379,6 @@ standalone: \
 		scripts/virtualenv.mk
 	cp -r $(addprefix $(BSP_DIR)/,$(filter-out build,$(shell ls $(BSP_DIR)))) $</bsp/
 
-	cp -r pip-cache $</
-
 	cp -r freedom-metal $</
 	find $</freedom-metal -name ".git*" | xargs rm -rf
 

README.md

@@ -240,6 +240,47 @@ git pull origin master
 git submodule update --init --recursive
 ```
 
+### Python ###
+
+Freedom E SDK includes a number of Python scripts used during the build process
+to parameterize the build of Freedom Metal to the target. The dependencies of
+these scripts are tracked in `requirements.txt`. Freedom E SDK manages its own
+virtualenv, but there are some options which allow users to configure the
+virtualenv to best suit your needs.
+
+#### Predownloading Python Dependencies ####
+
+By default, Freedom E SDK will download Python packages from the Python Package
+Index when it creates the virtualenv. If you prefer to download dependencies
+ahead-of-time, you can run
+
+```
+make pip-cache
+```
+
+to download all Python packages. This mechanism downloads all of the dependencies
+pre-compiled for all platforms and Python versions supported by Freedom E SDK, so
+if you're trying to bring up Freedom E SDK on a system without an internet connection
+you can create the "pip cache" and then copy it to the connectionless machine with
+Freedom E SDK.
+
+The location of the "pip cache" can be controlled with the environment variable
+`FREEDOM_E_SDK_PIP_CACHE_PATH`
+
+```
+export FREEDOM_E_SDK_PIP_CACHE_PATH=/path/to/pip-cache
+```
+
+#### Virtualenv Location ####
+
+By default, the virtualenv is created in the `venv` folder at the root of
+Freedom E SDK. To change the location of the virtualenv, set the environment
+variable `FREEDOM_E_SDK_VENV_PATH`
+
+```
+export FREEDOM_E_SDK_VENV_PATH=/path/to/venv
+```
+
 ### Using the Tools ###
 
 #### Building an Example ####

requirements.txt

@@ -9,4 +9,4 @@ MarkupSafe==1.1.1
 setuptools>=30.3.0
 wheel
 setuptools_scm>=3.3.1
-pip
+pip==20.0.1

scripts/download-python-packages.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import sys
+import subprocess
+from itertools import product
+
+if __name__ == "__main__":
+    requirements = sys.argv[1]
+    destination = sys.argv[2]
+
+    platforms = ["manylinux1_x86_64", "macosx_10_13_x86_64", "win32", "win_amd64"]
+    versions = ["35", "36", "37"]
+
+    tuples = product(platforms, versions)
+
+    base_cmd = ["python3", "-m", "pip", "download", "-d", destination, "-r", requirements]
+
+    subprocess.run(base_cmd + ["--platform", "any", "--no-deps"])
+
+    for platform, version in tuples:
+        subprocess.run(base_cmd + ["--platform", platform, "--python-version", version, "--only-binary=:all:"])
+
+    print(list(tuples))

scripts/virtualenv.mk

@@ -7,42 +7,63 @@ FREEDOM_E_SDK_VENV_PATH ?= venv
 .PHONY: virtualenv
 virtualenv: $(FREEDOM_E_SDK_VENV_PATH)/.stamp
 
-# Just in case the cached dependencies fail, users can install Python packages
-# from the Python Package Index by running `make virtualenv-from-pypi`
-
-.PHONY: virtualenv-from-pypi
-
-# invoke python3 pip in order to bypass 127 character limit in shebang
-virtualenv-from-pypi: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt
-	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --upgrade pip
-	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install -r requirements.txt
+.PHONY: clean-virtualenv
+clean-virtualenv:
+	-rm -rf $(FREEDOM_E_SDK_VENV_PATH)
 
-$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt
-	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links pip-cache --upgrade pip
-	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links pip-cache -r requirements.txt
-	touch $@
+# Freedom E SDK has two options for fetching Python dependencies:
+#
+#  1. Dependencies are fetched from the Python Package Index (PyPI) when the
+#     virtualenv is created.
+#  2. Dependencies are prefeteched from PyPI and stored in
+#      FREEDOM_E_SDK_PIP_CACHE_PATH, then installed from there at virtualenv
+#      creation time.
+#
+# By default, the pip-cache is created in the `pip-cache` folder at the root of
+# freedom-e-sdk. If you want the pip-cache to be placed elsewhere, set the
+# FREEDOM_E_SDK_PIP_CACHE_PATH environment variable.
+FREEDOM_E_SDK_PIP_CACHE_PATH ?= pip-cache
 
 $(FREEDOM_E_SDK_VENV_PATH)/bin/activate:
 	python3 -m venv $(FREEDOM_E_SDK_VENV_PATH)
+	. $@ && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install pip==20.0.1
 
-# The pip-cache directory holds a cache of all the Python package dependencies, being careful
-# that all are platform-indepentent.
-#
-# The result should be that when freedom-e-sdk is cloned, the virtualenv is created simply
-# by installing all cached dependencies from the submodule
-#
-# The pip-cache can be updated when requirements.txt changes by running `make update-pip-cache`,
-# commiting the changes to the submodule, and bumping the submodule.
+.PHONY: pip-cache
+pip-cache: $(FREEDOM_E_SDK_PIP_CACHE_PATH)/.stamp
 
-.PHONY: update-pip-cache
-update-pip-cache: requirements.txt
-	./pip-cache/download-python-dependencies.py $<
+$(FREEDOM_E_SDK_PIP_CACHE_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt
+	. $< && python3 scripts/download-python-packages.py requirements.txt $(FREEDOM_E_SDK_PIP_CACHE_PATH)
+	touch $@
 
 .PHONY: clean-pip-cache
 clean-pip-cache:
-	-rm -rf pip-cache/*.whl pip-cache *.tar.gz
+	-rm -r $(FREEDOM_E_SDK_PIP_CACHE_PATH)
 
-.PHONY: clean-virtualenv
-clean-virtualenv:
-	-rm -rf $(FREEDOM_E_SDK_VENV_PATH)
+# If FREEDOM_E_SDK_PIP_CACHE_PATH does not exist, fetch dependencies from PyPI
+ifeq ("$(wildcard $(FREEDOM_E_SDK_PIP_CACHE_PATH))","")
+
+# We invoke pip as a script in order to avoid depending on the shebang line, which might
+# exceed the 127-character limit depending on your environment.
+
+$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt
+	@echo "################################################################################" >&2
+	@echo "FREEDOM_E_SDK_PIP_CACHE not found, creating virtualenv from Python Package Index" >&2
+	@echo "You can pre-download Python packages by running 'make pip-cache'."                >&2
+	@echo "################################################################################" >&2
+	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install pip==20.0.1
+	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install -r requirements.txt
+	touch $@
+
+else # If FREEDOM_E_SDK_PIP_CACHE_PATH does exist, fetch dependences from there
+
+# We invoke pip as a script in order to avoid depending on the shebang line, which might
+# exceed the 127-character limit depending on your environment.
+
+$(FREEDOM_E_SDK_VENV_PATH)/.stamp: $(FREEDOM_E_SDK_VENV_PATH)/bin/activate requirements.txt
+	@echo "##################################################################" >&2
+	@echo "FREEDOM_E_SDK_PIP_CACHE found, creating virtualenv from $(FREEDOM_E_SDK_PIP_CACHE_PATH)" >&2
+	@echo "##################################################################" >&2
+	. $< && $(FREEDOM_E_SDK_VENV_PATH)/bin/python3 $(FREEDOM_E_SDK_VENV_PATH)/bin/pip install --no-index --find-links $(FREEDOM_E_SDK_PIP_CACHE_PATH) -r requirements.txt
+	touch $@
 
+endif # FREEDOM_E_SDK_PIP_CACHE_PATH

wit-manifest.json

@@ -14,11 +14,6 @@
         "name": "elf2hex",
         "source": "git@github.com:sifive/elf2hex.git"
     },
-    {
-        "commit": "e73944211dfc5b5493f6c1e19fe8bd0cd1464257",
-        "name": "pip-cache",
-        "source": "git@github.com:sifive/freedom-e-sdk-pip-cache.git"
-    },
     {
         "commit": "726d8761fba8108d4038c050dbd80d539766a11b",
         "name": "devicetree-overlay-generator",