From 3f2dd81967e47659f5376e00d2da3fed41625984 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 09:57:01 +0100 Subject: [PATCH 1/9] Correct URL for opam archive cache --- src_ext/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index 80279148c3b..cdaf7974ec1 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -144,7 +144,7 @@ has-archives: $(addprefix archives/, $(notdir $(URL_ocaml)) $(notdir $(URL_flexd ($(call FETCH,$(URL_PKG_$*),$(notdir $(URL_PKG_$*))) && mv $(notdir $(URL_PKG_$*)) archives/) define cache_url -https://opam.ocaml.org/2.0/cache/md5/$(shell echo $(MD5_$(2)$(1)) | cut -c -2)/$(MD5_$(2)$(1)) +https://opam.ocaml.org/cache/md5/$(shell echo $(MD5_$(2)$(1)) | cut -c -2)/$(MD5_$(2)$(1)) endef define get_from_cache From 86ce8cd1f1c3ea30e60ea057de24a5af9682119d Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 09:57:14 +0100 Subject: [PATCH 2/9] Allow make -C src_ext cache-archives to use cache Unblocks CI when upstream tarballs aren't available. --- src_ext/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index cdaf7974ec1..ea00ed4cbc6 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -136,12 +136,12 @@ has-archives: $(addprefix archives/, $(notdir $(URL_ocaml)) $(notdir $(URL_flexd %.cache: @mkdir -p archives @[ -e archives/$(notdir $(URL_$*)) ] || \ - ($(call FETCH,$(URL_$*),$(notdir $(URL_$*))) && mv $(notdir $(URL_$*)) archives/) + ( ( $(call FETCH,$(URL_$*),$(call ARCHIVE_FILE,$*)) || $(call get_from_cache,$*) ) && mv $(call ARCHIVE_FILE,$*) archives/$(notdir $(URL_$*)) ) %.pkgcache: @mkdir -p archives @[ -e archives/$(notdir $(URL_PKG_$*)) ] || \ - ($(call FETCH,$(URL_PKG_$*),$(notdir $(URL_PKG_$*))) && mv $(notdir $(URL_PKG_$*)) archives/) + ( ( $(call FETCH,$(URL_PKG_$*),$(call ARCHIVE_FILE,$*,PKG_)) || $(call get_from_cache,$*,PKG_) ) && mv $(call ARCHIVE_FILE,$*,PKG_) archives/$(notdir $(URL_PKG_$*)) ) define cache_url https://opam.ocaml.org/cache/md5/$(shell echo $(MD5_$(2)$(1)) | cut -c -2)/$(MD5_$(2)$(1)) From 148d6dc7531eca51ae850b2c61ae5a06f638e345 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 10:04:29 +0100 Subject: [PATCH 3/9] Display OS information on AppVeyor logs --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index d5bd45e4f3f..f894b31217e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -31,6 +31,7 @@ cache: - C:\projects\opam\src_ext\archives init: + - systeminfo 2>nul | findstr /B /C:"OS Name" /C:"OS Version" - "echo System architecture: %PLATFORM%" install: From 14f5549fae1edef221c0538053aa14579156212e Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 13:27:58 +0100 Subject: [PATCH 4/9] Synchronise the two MD5 mechanisms in src_ext --- shell/md5check.ml | 2 +- src_ext/Makefile | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/shell/md5check.ml b/shell/md5check.ml index 983fcafe091..64a4225fe0e 100644 --- a/shell/md5check.ml +++ b/shell/md5check.ml @@ -11,7 +11,7 @@ let md5_of_file = let () = if md5 <> md5_of_file then ( Printf.eprintf - "MD5 for %s differ:\n\ + "MD5 for %s differs:\n\ \ expected: %s\n\ \ actual: %s\n" file md5 md5_of_file; diff --git a/src_ext/Makefile b/src_ext/Makefile index ea00ed4cbc6..f98e94cc6ee 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -56,7 +56,16 @@ ifdef OCAML # Portable md5check MD5CHECK = $(OCAML) ../shell/md5check.ml $(1) $(2) else -MD5CHECK = test "`md5sum $(1) | sed -e 's/^[^a-f0-9]*\([a-f0-9]*\).*/\1/'`" = "$(2)" || (rm $(1) && false) +MD5CHECK = { \ + sum=`md5sum $(1) | sed -e 's/^[^a-f0-9]*\([a-f0-9]*\).*/\1/'`; \ + { test "$$sum" = "$(2)" && echo '$(1) has the expected MD5.'; } || \ + { rm -f $(1); \ + echo 'MD5 for $(1) differs:'; \ + echo ' expected: $(2)'; \ + echo " actual: $$sum"; \ + false; \ + }; \ +} endif lib-ext: clone ensure-seq-patched.stamp From 043e04728069535ee8c999a3a4cb8d34ff3edcb2 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 13:32:27 +0100 Subject: [PATCH 5/9] Overhaul downloading code in src_ext/Makefile MD5 sums were not checked in make -C src_ext cache-archives (so CI archives were not verified). Error reporting improved - it's now clear which tarballs were skipped on error. Code for downloading archives between lib-ext/lib-pkg and cache-archives merged. --- src_ext/Makefile | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index f98e94cc6ee..69c3ade7e33 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -142,39 +142,35 @@ cache-archives: $(SRC_EXTS:=.cache) $(PKG_EXTS:=.pkgcache) ocaml.cache flexdll.c has-archives: $(addprefix archives/, $(notdir $(URL_ocaml)) $(notdir $(URL_flexdll)) $(ARCHIVES) $(filter-out $(ARCHIVES), $(foreach pkg,$(PKG_EXTS), $(notdir $(URL_PKG_$(pkg)))))) @ +define cache_url +https://opam.ocaml.org/cache/md5/$(shell echo $(MD5_$(2)$(1)) | cut -c -2)/$(MD5_$(2)$(1)) +endef + +GET_ARCHIVE=\ + { { { $(call FETCH,$(URL_$(2)$(1)),$(call ARCHIVE_FILE,$(1),$(2))) && $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))); } \ + || { echo 'Failed to download $(URL_$(2)$(1))'; false; }; } || \ + { { $(call FETCH,$(call cache_url,$(1),$(2)),$(call ARCHIVE_FILE,$(1),$(2))) && $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))); } \ + || { echo 'Failed to download $(1) from opam cache'; false; }; }; } + %.cache: @mkdir -p archives - @[ -e archives/$(notdir $(URL_$*)) ] || \ - ( ( $(call FETCH,$(URL_$*),$(call ARCHIVE_FILE,$*)) || $(call get_from_cache,$*) ) && mv $(call ARCHIVE_FILE,$*) archives/$(notdir $(URL_$*)) ) + @test -e archives/$(notdir $(URL_$*)) || \ + { $(call GET_ARCHIVE,$*) && mv $(call ARCHIVE_FILE,$*) archives/$(notdir $(URL_$*)); } %.pkgcache: @mkdir -p archives - @[ -e archives/$(notdir $(URL_PKG_$*)) ] || \ - ( ( $(call FETCH,$(URL_PKG_$*),$(call ARCHIVE_FILE,$*,PKG_)) || $(call get_from_cache,$*,PKG_) ) && mv $(call ARCHIVE_FILE,$*,PKG_) archives/$(notdir $(URL_PKG_$*)) ) - -define cache_url -https://opam.ocaml.org/cache/md5/$(shell echo $(MD5_$(2)$(1)) | cut -c -2)/$(MD5_$(2)$(1)) -endef - -define get_from_cache -{ $(call FETCH,$(call cache_url,$(1),$(2)),$(MD5_$(2)$(1))) && \ - mv $(MD5_$(2)$(1)) $(call ARCHIVE_FILE,$(1),$(2)) && \ - $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))); } -endef + @test -e archives/$(notdir $(URL_$*)) || \ + { $(call GET_ARCHIVE,$*,PKG_) && mv $(call ARCHIVE_FILE,$*,PKG_) archives/$(notdir $(URL_PKG_$*)); } %.download: Makefile.sources @$(call DOWNLOAD_COOKIE,$*,,PKG_,,pkg) [ -e $(call ARCHIVE_FILE,$*) ] || \ - cp archives/$(notdir $(URL_$*)) $(call ARCHIVE_FILE,$*) 2>/dev/null || \ - { $(call FETCH,$(URL_$*),$(call ARCHIVE_FILE,$*)) && $(call MD5CHECK,$(call ARCHIVE_FILE,$*),$(MD5_$*)); } || \ - $(call get_from_cache,$*) + cp archives/$(notdir $(URL_$*)) $(call ARCHIVE_FILE,$*) 2>/dev/null || $(call GET_ARCHIVE,$*) %.pkgdownload: Makefile.sources @$(call DOWNLOAD_COOKIE,$*,PKG_,,pkg) [ -e $(call ARCHIVE_FILE,$*,PKG_) ] || \ - cp archives/$(notdir $(URL_PKG_$*)) $(call ARCHIVE_FILE,$*,PKG_) 2>/dev/null || \ - { $(call FETCH,$(URL_PKG_$*),$(call ARCHIVE_FILE,$*,PKG_)) && $(call MD5CHECK,$(call ARCHIVE_FILE,$*,PKG_),$(MD5_PKG_$*)); } || \ - $(call get_from_cache,$*,PKG_) + cp archives/$(notdir $(URL_PKG_$*)) $(call ARCHIVE_FILE,$*,PKG_) 2>/dev/null || $(call GET_ARCHIVE,$*,PKG_) %.stamp: %.download mkdir -p tmp-$* From 74441e71ef05ec64b1428abdcbce6b33b21beb35 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 13:34:24 +0100 Subject: [PATCH 6/9] Correct flexdll tarball checksum --- src_ext/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index 69c3ade7e33..93efe7f5b10 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -15,7 +15,7 @@ URL_ocaml = https://caml.inria.fr/pub/distrib/ocaml-4.09/ocaml-4.09.1.tar.gz MD5_ocaml = 1a7fc32aebbad6cb821ef26a396f78e7 URL_flexdll = https://github.com/alainfrisch/flexdll/archive/0.38.tar.gz -MD5_flexdll = c0fcfd7554b68547435fb69db8cde260 +MD5_flexdll = dca52bce081c8f19e9cd78219458a816 ifndef FETCH ifneq ($(shell command -v curl 2>/dev/null),) From f4c134a22da77c48ed4324762d74f953c311e33f Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 14:20:10 +0100 Subject: [PATCH 7/9] Display a message when the opam cache used --- src_ext/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index 93efe7f5b10..cb59addd2ff 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -149,7 +149,7 @@ endef GET_ARCHIVE=\ { { { $(call FETCH,$(URL_$(2)$(1)),$(call ARCHIVE_FILE,$(1),$(2))) && $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))); } \ || { echo 'Failed to download $(URL_$(2)$(1))'; false; }; } || \ - { { $(call FETCH,$(call cache_url,$(1),$(2)),$(call ARCHIVE_FILE,$(1),$(2))) && $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))); } \ + { { $(call FETCH,$(call cache_url,$(1),$(2)),$(call ARCHIVE_FILE,$(1),$(2))) && $(call MD5CHECK,$(call ARCHIVE_FILE,$(1),$(2)),$(MD5_$(2)$(1))) && echo 'Warning: downloaded $(URL_$(2)$(1)) from opam cache'; } \ || { echo 'Failed to download $(1) from opam cache'; false; }; }; } %.cache: From 39328e834b65433cdb7b5d9f9773370540dd5dae Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 15:23:11 +0100 Subject: [PATCH 8/9] Revert "Install GNU make 4.2 for the cold test" This reverts commit 4869ee61744f57fcd6fa1b8c5671c4806e7edf11. --- .travis-ci.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.travis-ci.sh b/.travis-ci.sh index a9974b073dc..705fa77fdf8 100755 --- a/.travis-ci.sh +++ b/.travis-ci.sh @@ -116,18 +116,7 @@ wrap-install-commands: [] wrap-remove-commands: [] EOF - if [[ $COLD -eq 1 ]] ; then - if [ ! -x ~/local/bin/make ] ; then - wget http://ftpmirror.gnu.org/gnu/make/make-4.2.tar.gz - tar -xzf make-4.2.tar.gz - mkdir make-4.2-build - cd make-4.2-build - ../make-4.2/configure --prefix ~/local - make - make install - cd .. - fi - else + if [[ $COLD -ne 1 ]] ; then if [[ $TRAVIS_OS_NAME = "osx" && -n $EXTERNAL_SOLVER ]] ; then rvm install ruby-2.3.3 rvm --default use 2.3.3 From 40c4ab8921cec37a06ad0c19efe792d22b2b27a2 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Tue, 7 Jul 2020 15:28:59 +0100 Subject: [PATCH 9/9] Correct typo src_ext/Makefile Apparently GNU make 4 is more forgiving of poor syntax... --- src_ext/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_ext/Makefile b/src_ext/Makefile index cb59addd2ff..52d3ceb6081 100644 --- a/src_ext/Makefile +++ b/src_ext/Makefile @@ -27,7 +27,7 @@ endif # Shorthand for designating that lib-ext and lib-pkg use the same version of a library PKG_SAME = $(eval $(call PKG_SAME_DEFS,$(1))) -define PKG_SAME_DEFS = +define PKG_SAME_DEFS URL_PKG_$(1) = $(URL_$(1)) MD5_PKG_$(1) = $(MD5_$(1)) endef