From c8de1754c46f2e538fab1ed137a89d436d5667a7 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 16 Jul 2021 14:29:00 +0300 Subject: [PATCH 1/2] Refactor Makefile to improve readability --- Makefile | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 5694b3a556..af48dce42b 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ SHELL:=bash OWNER?=jupyter # Need to list the images in build dependency order -ALL_IMAGES:=base-notebook \ +ALL_IMAGES:= \ + base-notebook \ minimal-notebook \ r-notebook \ scipy-notebook \ @@ -19,6 +20,8 @@ ALL_IMAGES:=base-notebook \ # Enable BuildKit for Docker build export DOCKER_BUILDKIT:=1 + + # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html help: @echo "jupyter/docker-stacks" @@ -27,29 +30,34 @@ help: @echo @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + + build/%: DARGS?= build/%: ## build the latest image for a stack docker build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest --build-arg OWNER=$(OWNER) ./$(notdir $@) @echo -n "Built image size: " @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" +build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks +build-test-all: $(foreach I, $(ALL_IMAGES), build/$(I) test/$(I)) ## build and test all stacks + -build-all: $(foreach I,$(ALL_IMAGES), build/$(I) ) ## build all stacks -build-test-all: $(foreach I,$(ALL_IMAGES), build/$(I) test/$(I) ) ## build and test all stacks check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py -check-outdated-all: $(foreach I,$(ALL_IMAGES), check-outdated/$(I) ) ## check all the stacks for outdated conda packages +check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated conda packages + -cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm) +cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm) cont-stop-all: ## stop all containers @echo "Stopping all containers ..." -docker stop -t0 $(shell docker ps -a -q) 2> /dev/null - cont-rm-all: ## remove all containers @echo "Removing all containers ..." -docker rm --force $(shell docker ps -a -q) 2> /dev/null + + dev/%: ARGS?= dev/%: DARGS?=-e JUPYTER_ENABLE_LAB=yes dev/%: PORT?=8888 @@ -59,48 +67,53 @@ dev/%: ## run a foreground container for a stack dev-env: ## install libraries required to build docs and run tests @pip install -r requirements-dev.txt + + docs: ## build HTML documentation sphinx-build docs/ docs/_build/ + + hook/%: WIKI_PATH?=../wiki hook/%: ## run post-build hooks for an image python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \ python3 -m tagging.create_manifests --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)" +hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images -hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images -img-clean: img-rm-dang img-rm ## clean dangling and jupyter images +img-clean: img-rm-dang img-rm ## clean dangling and jupyter images img-list: ## list jupyter images @echo "Listing $(OWNER) images ..." docker images "$(OWNER)/*" - img-rm: ## remove jupyter images @echo "Removing $(OWNER) images ..." -docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null - img-rm-dang: ## remove dangling images (tagged None) @echo "Removing dangling images ..." -docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null + + pre-commit-all: ## run pre-commit hook on all files @pre-commit run --all-files || (printf "\n\n\n" && git --no-pager diff --color=always) - pre-commit-install: ## set up the git hook scripts @pre-commit --version @pre-commit install + + pull/%: DARGS?= pull/%: ## pull a jupyter image docker pull $(DARGS) $(OWNER)/$(notdir $@) - -pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images +pull-all: $(foreach I, $(ALL_IMAGES), pull/$(I)) ## pull all images push/%: DARGS?= push/%: ## push all tags for a jupyter image docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@) +push-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images + -push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images run/%: DARGS?= run/%: ## run a bash in interactive mode in a stack @@ -110,8 +123,10 @@ run-sudo/%: DARGS?= run-sudo/%: ## run a bash in interactive mode as root in a stack docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL) + + test/%: ## run tests against a stack (only common tests or common tests + specific tests) @if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \ else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi -test-all: $(foreach I,$(ALL_IMAGES),test/$(I)) ## test all stacks +test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks From 667c4f91d106cea41b2ec7e0800704a3e3bb57be Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 16 Jul 2021 15:25:32 +0300 Subject: [PATCH 2/2] Do not import packages which are not importable --- test/test_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_packages.py b/test/test_packages.py index 0aa12edc45..01240affde 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -69,6 +69,8 @@ "r-irkernel", "unixodbc", "bzip2", + "openssl", + "ca-certificates", ]