From eaeed10b985233b117526f634726f84b2d0f056f Mon Sep 17 00:00:00 2001 From: Raja Boujbel Date: Wed, 17 Jul 2024 15:16:47 +0200 Subject: [PATCH 1/2] reftest: add admin cache test --- .github/scripts/cygwin.cmd | 2 + master_changes.md | 1 + tests/reftests/admin-cache.test | 416 ++++++++++++++++++++++++++++++++ tests/reftests/admin.test | 4 + tests/reftests/dune.inc | 18 ++ 5 files changed, 441 insertions(+) create mode 100644 tests/reftests/admin-cache.test diff --git a/.github/scripts/cygwin.cmd b/.github/scripts/cygwin.cmd index d0e97725fc3..934e6857ce5 100644 --- a/.github/scripts/cygwin.cmd +++ b/.github/scripts/cygwin.cmd @@ -93,6 +93,8 @@ if "%1" equ "x86_64-pc-cygwin" ( :: libicu-devel is needed until an alternative to the uconv call in MungeSymlinks :: is found set CYGWIN_PACKAGES=make,patch,curl,diffutils,tar,unzip,git,gcc-g++,libicu-devel%CYGWIN_PACKAGES% +:: xxd is needed for reftests +set CYGWIN_PACKAGES=xxd,%CYGWIN_PACKAGES% :: wget is needed for download.test OPAMFETCH/OPAMCURL testing set CYGWIN_PACKAGES=wget,%CYGWIN_PACKAGES% diff --git a/master_changes.md b/master_changes.md index d6f409b59c8..27aad71dce4 100644 --- a/master_changes.md +++ b/master_changes.md @@ -125,6 +125,7 @@ users) * Add test for filter operators in opam file [#5642 @rjbou] * Update init test to make it no repo [#5327 @rjbou] * Add a test in admin cache for hash cache [#6103 @rjbou] + * Add admin cache test [#6068 @rjbou] ### Engine diff --git a/tests/reftests/admin-cache.test b/tests/reftests/admin-cache.test new file mode 100644 index 00000000000..b87ee3acd1f --- /dev/null +++ b/tests/reftests/admin-cache.test @@ -0,0 +1,416 @@ +N0REP0 +### mv REPO OPER +### opam repository set-url default OPER/ --set-default +[default] Initialised +### : create repo +### +opam-version: "1.2" +### +opam-version: "2.0" +### +opam-version: "2.0" +### +opam-version: "2.0" +### +opam-version: "2.0" +### +for op in `ls packages/*/*/opam`; do + grep -q synopsis $op || echo 'synopsis: "A word"' >> $op + grep -q description $op || echo 'description: "Two words."' >> $op + grep -q authors $op || echo 'authors: "the testing team"' >> $op + grep -q homepage $op || echo 'homepage: "egapemoh"' >> $op + grep -q maintainer $op || echo 'maintainer: "maint@tain.er"' >> $op + grep -q license $op || echo 'license: "MIT"' >> $op + grep -q dev-repo $op || echo 'dev-repo: "hg+https://pkg@op.am"' >> $op + grep -q bug-reports $op || echo 'bug-reports: "https://nobug"' >> $op +done +### sh add-content.sh +### : add archives +### tar czf arch-ipsum.tgz add-content.sh +### tar czf arch-dolor.tgz repo +### tar czf arch-sit.tgz OPAM/config +### tar czf arch-amet.tgz OPAM/repo/repos-config +### +for n in $@; do + nv=$n.1 + arch=arch-$n.tgz + file="packages/$n/$nv/opam" + echo "url {" >> $file + echo "src: \"$arch\"" >> $file + MD5=$(openssl md5 $arch | cut -d' ' -f2) + echo "checksum: \"md5=$MD5\"" >> $file + echo "}" >> $file +done +### sh add-urls.sh ipsum dolor sit amet +### opam admin add-hashes sha256 +### opam admin add-hashes sha512 +[file://${BASEDIR}/arch-amet.tgz] found in cache +[file://${BASEDIR}/arch-dolor.tgz] found in cache +[file://${BASEDIR}/arch-ipsum.tgz] found in cache +[file://${BASEDIR}/arch-sit.tgz] found in cache +### rm -rf cache +### : computes hashes +### openssl md5 arch-ipsum.tgz | '.*= ' -> '' >$ IP_MD5 +### sh -c "echo '$IP_MD5' | cut -c 1-2" >$ PRE_IP_MD5 +### openssl sha256 arch-ipsum.tgz | '.*= ' -> '' >$ IP_SHA256 +### sh -c "echo '$IP_SHA256' | cut -c 1-2" >$ PRE_IP_SHA256 +### openssl sha512 arch-ipsum.tgz | '.*= ' -> '' >$ IP_SHA512 +### sh -c "echo '$IP_SHA512' | cut -c 1-2" >$ PRE_IP_SHA512 +### openssl md5 arch-dolor.tgz | '.*= ' -> '' >$ DL_MD5 +### sh -c "echo '$DL_MD5' | cut -c 1-2" >$ PRE_DL_MD5 +### openssl sha256 arch-dolor.tgz | '.*= ' -> '' >$ DL_SHA256 +### sh -c "echo '$DL_SHA256' | cut -c 1-2" >$ PRE_DL_SHA256 +### openssl sha512 arch-dolor.tgz | '.*= ' -> '' >$ DL_SHA512 +### sh -c "echo '$DL_SHA512' | cut -c 1-2" >$ PRE_DL_SHA512 +### openssl md5 arch-sit.tgz | '.*= ' -> '' >$ ST_MD5 +### sh -c "echo '$ST_MD5' | cut -c 1-2" >$ PRE_ST_MD5 +### openssl sha256 arch-sit.tgz | '.*= ' -> '' >$ ST_SHA256 +### sh -c "echo '$ST_SHA256' | cut -c 1-2" >$ PRE_ST_SHA256 +### openssl sha512 arch-sit.tgz | '.*= ' -> '' >$ ST_SHA512 +### sh -c "echo '$ST_SHA512' | cut -c 1-2" >$ PRE_ST_SHA512 +### openssl md5 arch-amet.tgz | '.*= ' -> '' >$ MT_MD5 +### sh -c "echo '$MT_MD5' | cut -c 1-2" >$ PRE_MT_MD5 +### openssl sha256 arch-amet.tgz | '.*= ' -> '' >$ MT_SHA256 +### sh -c "echo '$MT_SHA256' | cut -c 1-2" >$ PRE_MT_SHA256 +### openssl sha512 arch-amet.tgz | '.*= ' -> '' >$ MT_SHA512 +### sh -c "echo '$MT_SHA512' | cut -c 1-2" >$ PRE_MT_SHA512 +### +pkg=$1 +kind=$2 +hash=$3 + +pre=`echo $hash | cut -c 1-2` +arch=cache/$kind/$pre/$hash +msg="$pkg : $kind :" + +if [ -L $arch ]; then + link=`readlink "$arch"` + hash=$(basename $(dirname $(dirname $link))) + real=`realpath -mLP $arch` + if [ -f $real ]; then + abs="" + else + abs=" (absent)" + fi + echo "$msg exists, link to $hash$abs" +elif [ -f "$arch" ] ; then + echo "$msg exists" +else + echo "$msg not found" +fi +### +sh analysi.sh ipsum md5 $IP_MD5 +sh analysi.sh ipsum sha256 $IP_SHA256 +sh analysi.sh ipsum sha512 $IP_SHA512 +echo -- +sh analysi.sh dolor md5 $DL_MD5 +sh analysi.sh dolor sha256 $DL_SHA256 +sh analysi.sh dolor sha512 $DL_SHA512 +echo -- +sh analysi.sh sit md5 $ST_MD5 +sh analysi.sh sit sha256 $ST_SHA256 +sh analysi.sh sit sha512 $ST_SHA512 +echo -- +sh analysi.sh amet md5 $MT_MD5 +sh analysi.sh amet sha256 $MT_SHA256 +sh analysi.sh amet sha512 $MT_SHA512 +### : create cache : +### opam admin cache | unordered +[dolor.1] synchronised (file://${BASEDIR}/arch-dolor.tgz) +[ipsum.1] synchronised (file://${BASEDIR}/arch-ipsum.tgz) +[sit.1] synchronised (file://${BASEDIR}/arch-sit.tgz) +[amet.1] synchronised (file://${BASEDIR}/arch-amet.tgz) +Adding cache to ${BASEDIR}/repo... +Done. +### ::::::::::::::::::::::::::: +### :I: check hashes presence : +### ::::::::::::::::::::::::::: +### sh cache-analysi.sh +ipsum : md5 : exists, link to sha512 +ipsum : sha256 : exists, link to sha512 +ipsum : sha512 : exists +-- +dolor : md5 : exists, link to sha512 +dolor : sha256 : exists, link to sha512 +dolor : sha512 : exists +-- +sit : md5 : exists, link to sha512 +sit : sha256 : exists, link to sha512 +sit : sha512 : exists +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### :::::::::::::::::::::::::::::::::::::::::: +### :II: remove weakest hashes and recompute : +### :::::::::::::::::::::::::::::::::::::::::: +### # in cache, the archiv is the strongest one +### rm -rf cache/md5 cache/sha256 +### opam admin cache | unordered +Done. +### sh cache-analysi.sh +ipsum : md5 : not found +ipsum : sha256 : not found +ipsum : sha512 : exists +-- +dolor : md5 : not found +dolor : sha256 : not found +dolor : sha512 : exists +-- +sit : md5 : not found +sit : sha256 : not found +sit : sha512 : exists +-- +amet : md5 : not found +amet : sha256 : not found +amet : sha512 : exists +### opam admin cache | unordered +Done. +### sh cache-analysi.sh +ipsum : md5 : not found +ipsum : sha256 : not found +ipsum : sha512 : exists +-- +dolor : md5 : not found +dolor : sha256 : not found +dolor : sha512 : exists +-- +sit : md5 : not found +sit : sha256 : not found +sit : sha512 : exists +-- +amet : md5 : not found +amet : sha256 : not found +amet : sha512 : exists +### ::::::::::::::::::::::::::::::::::::::::::: +### :III: remove strongest hash and recompute : +### ::::::::::::::::::::::::::::::::::::::::::: +### rm -rf cache +### opam admin cache | unordered +[ipsum.1] synchronised (file://${BASEDIR}/arch-ipsum.tgz) +[dolor.1] synchronised (file://${BASEDIR}/arch-dolor.tgz) +[sit.1] synchronised (file://${BASEDIR}/arch-sit.tgz) +[amet.1] synchronised (file://${BASEDIR}/arch-amet.tgz) +Done. +### sh cache-analysi.sh +ipsum : md5 : exists, link to sha512 +ipsum : sha256 : exists, link to sha512 +ipsum : sha512 : exists +-- +dolor : md5 : exists, link to sha512 +dolor : sha256 : exists, link to sha512 +dolor : sha512 : exists +-- +sit : md5 : exists, link to sha512 +sit : sha256 : exists, link to sha512 +sit : sha512 : exists +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### rm -rf cache/sha512 +### sh cache-analysi.sh +ipsum : md5 : exists, link to sha512 (absent) +ipsum : sha256 : exists, link to sha512 (absent) +ipsum : sha512 : not found +-- +dolor : md5 : exists, link to sha512 (absent) +dolor : sha256 : exists, link to sha512 (absent) +dolor : sha512 : not found +-- +sit : md5 : exists, link to sha512 (absent) +sit : sha256 : exists, link to sha512 (absent) +sit : sha512 : not found +-- +amet : md5 : exists, link to sha512 (absent) +amet : sha256 : exists, link to sha512 (absent) +amet : sha512 : not found +### opam admin cache | unordered +[dolor.1] synchronised (file://${BASEDIR}/arch-dolor.tgz) +[ipsum.1] synchronised (file://${BASEDIR}/arch-ipsum.tgz) +[sit.1] synchronised (file://${BASEDIR}/arch-sit.tgz) +[amet.1] synchronised (file://${BASEDIR}/arch-amet.tgz) +Done. +### sh cache-analysi.sh +ipsum : md5 : exists, link to sha512 +ipsum : sha256 : exists, link to sha512 +ipsum : sha512 : exists +-- +dolor : md5 : exists, link to sha512 +dolor : sha256 : exists, link to sha512 +dolor : sha512 : exists +-- +sit : md5 : exists, link to sha512 +sit : sha256 : exists, link to sha512 +sit : sha512 : exists +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### :::::::::::::::::::::::::::::::::::: +### :IV: add a other hashes afterwards : +### :::::::::::::::::::::::::::::::::::: +### : ipsum : only md5 then adding sha512 +### : dolor : md5 & sha256 then adding sha512 +### : sit : sha256 then adding md5 & sha512 +### +regexp=$1 +file=$2 +sed "$regexp" "$file" > "$file.tmp" +mv "$file.tmp" "$file" +### sh sedi.sh '/"sha512=.*/d' packages/ipsum/ipsum.1/opam +### sh sedi.sh '/"sha256=.*/d' packages/ipsum/ipsum.1/opam +### sh sedi.sh '/"sha512=.*/d' packages/dolor/dolor.1/opam +### sh sedi.sh '/"md5=.*/d' packages/sit/sit.1/opam +### sh sedi.sh '/"sha512=.*/d' packages/sit/sit.1/opam +### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 +url.src: file://${BASEDIR}/arch-ipsum.tgz +url.checksum: md5=IP_MD5 +### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256 | "${DL_SHA512}" -> DL_SHA512 +url.src: file://${BASEDIR}/arch-dolor.tgz +url.checksum: md5=DL_MD5, sha256=DL_SHA256 +### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_MD5}" -> ST_MD5 | "${ST_SHA256}" -> ST_SHA256 | "${ST_SHA512}" -> ST_SHA512 +url.src: file://${BASEDIR}/arch-sit.tgz +url.checksum: sha256=ST_SHA256 +### rm -rf cache +### opam admin cache | unordered +[dolor.1] synchronised (file://${BASEDIR}/arch-dolor.tgz) +[ipsum.1] synchronised (file://${BASEDIR}/arch-ipsum.tgz) +[sit.1] synchronised (file://${BASEDIR}/arch-sit.tgz) +[amet.1] synchronised (file://${BASEDIR}/arch-amet.tgz) +Done. +### sh cache-analysi.sh +ipsum : md5 : exists +ipsum : sha256 : not found +ipsum : sha512 : not found +-- +dolor : md5 : exists, link to sha256 +dolor : sha256 : exists +dolor : sha512 : not found +-- +sit : md5 : not found +sit : sha256 : exists +sit : sha512 : not found +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### : add hashes +### opam admin add-hashes sha512 --package ipsum +### opam admin add-hashes sha512 --package dolor +### opam admin add-hashes md5 --package sit +[file://${BASEDIR}/arch-sit.tgz] downloaded from file://${BASEDIR}/cache +### opam admin add-hashes sha512 --package sit +### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 | "${IP_SHA512}" -> IP_SHA512 +url.src: file://${BASEDIR}/arch-ipsum.tgz +url.checksum: md5=IP_MD5, sha512=IP_SHA512 +### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256 | "${DL_SHA512}" -> DL_SHA512 +url.src: file://${BASEDIR}/arch-dolor.tgz +url.checksum: md5=DL_MD5, sha256=DL_SHA256, sha512=DL_SHA512 +### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_MD5}" -> ST_MD5 | "${ST_SHA256}" -> ST_SHA256 | "${ST_SHA512}" -> ST_SHA512 +url.src: file://${BASEDIR}/arch-sit.tgz +url.checksum: sha256=ST_SHA256, md5=ST_MD5, sha512=ST_SHA512 +### sh cache-analysi.sh +ipsum : md5 : exists +ipsum : sha256 : not found +ipsum : sha512 : not found +-- +dolor : md5 : exists, link to sha256 +dolor : sha256 : exists +dolor : sha512 : not found +-- +sit : md5 : not found +sit : sha256 : exists +sit : sha512 : not found +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### : cache +### opam admin cache | unordered +[dolor.1] synchronised (cached) +[ipsum.1] synchronised (cached) +[sit.1] synchronised (cached) +Done. +### sh cache-analysi.sh +ipsum : md5 : exists +ipsum : sha256 : not found +ipsum : sha512 : not found +-- +dolor : md5 : exists, link to sha256 +dolor : sha256 : exists +dolor : sha512 : not found +-- +sit : md5 : not found +sit : sha256 : exists +sit : sha512 : not found +-- +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 +amet : sha512 : exists +### :::::::::::::::::: +### :V: corrupted md5: +### :::::::::::::::::: +### +opam-version: "2.0" +### +4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa200a8284bf36e8e4b55b35f427593d849676da0d1555d8360fb5f07fea2 +### +4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa202a8284bf36e8e4b55b35f427593d849676da0d1d55d8360fb5f07fea2 +### xxd -r -p hex1 arch-lorem.tgz +### xxd -r -p hex2 arch-evil.tgz +### openssl md5 arch-lorem.tgz | '.*= ' -> '' >$ LR_MD5 +### sh -c "echo '$LR_MD5' | cut -c 1-2" >$ PRE_LR_MD5 +### openssl sha256 arch-lorem.tgz | '.*= ' -> '' >$ LR_SHA256 +### sh -c "echo '$LR_SHA256' | cut -c 1-2" >$ PRE_LR_SHA256 +### openssl sha512 arch-lorem.tgz | '.*= ' -> '' >$ LR_SHA512 +### sh -c "echo '$LR_SHA512' | cut -c 1-2" >$ PRE_LR_SHA512 +### openssl md5 arch-evil.tgz | "${LR_MD5}" -> LR_MD5 +MD5(arch-evil.tgz)= LR_MD5 +### sh add-urls.sh lorem +### opam admin cache | unordered +[dolor.1] synchronised (cached) +[ipsum.1] synchronised (cached) +[sit.1] synchronised (cached) +[lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz) +Done. +### opam admin add-hashes sha256 --package lorem +### +sh analysi.sh lorem md5 $LR_MD5 +sh analysi.sh lorem sha256 $LR_SHA256 +sh analysi.sh lorem sha512 $LR_SHA512 +### sh cache-analysi.sh +lorem : md5 : exists +lorem : sha256 : not found +lorem : sha512 : not found +### opam admin cache | unordered +[lorem.1] synchronised (cached) +[dolor.1] synchronised (cached) +[ipsum.1] synchronised (cached) +[sit.1] synchronised (cached) +Done. +### sh cache-analysi.sh +lorem : md5 : exists +lorem : sha256 : not found +lorem : sha512 : not found +### mv arch-evil.tgz cache/md5/$PRE_LR_MD5/$LR_MD5 +### opam admin add-hashes sha512 --package lorem +### sh cache-analysi.sh +lorem : md5 : exists +lorem : sha256 : not found +lorem : sha512 : not found +### opam admin cache | "${LR_MD5}" -> LR_MD5 | "${LR_SHA256}" -> LR_SHA256 | "${LR_SHA512}" -> LR_SHA512 +[ERROR] Conflicting file hashes, or broken or compromised cache! + - sha512=LR_SHA512 (MISMATCH) + - sha256=LR_SHA256 (MISMATCH) + - md5=LR_MD5 (match) + +[dolor.1] synchronised (cached) +[ipsum.1] synchronised (cached) +[sit.1] synchronised (cached) +[lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz) +Done. +### sh cache-analysi.sh +lorem : md5 : exists, link to sha512 +lorem : sha256 : exists, link to sha512 +lorem : sha512 : exists diff --git a/tests/reftests/admin.test b/tests/reftests/admin.test index 1c5089202d9..009d1a0222c 100644 --- a/tests/reftests/admin.test +++ b/tests/reftests/admin.test @@ -651,6 +651,10 @@ opam-version: "2.0" ### opam-cat packages/with-xf/with-xf.2/opam | "${XF_MD5}" -> "good-md5" | "${XF_SHA256}" -> "good-sha" extra-files: ["missing-hash" "md5=good-md5"] opam-version: "2.0" +### :::::::::::::::::::::::::::::::::::::::::::::::::::::: +### :: Cache is tested in admin-cache.test :: +### :::::::::::::::::::::::::::::::::::::::::::::::::::::: +### : ### :::::::::::::::::::::: ### :VII: Specific cases : ### :::::::::::::::::::::: diff --git a/tests/reftests/dune.inc b/tests/reftests/dune.inc index cf0ad125d9c..6cad84a211a 100644 --- a/tests/reftests/dune.inc +++ b/tests/reftests/dune.inc @@ -1,4 +1,22 @@ +(rule + (alias reftest-admin-cache) + (action + (diff admin-cache.test admin-cache.out))) + +(alias + (name reftest) + (deps (alias reftest-admin-cache))) + +(rule + (targets admin-cache.out) + (deps root-N0REP0) + (package opam) + (action + (with-stdout-to + %{targets} + (run ./run.exe %{exe:../../src/client/opamMain.exe.exe} %{dep:admin-cache.test} %{read-lines:testing-env})))) + (rule (alias reftest-admin) (action From 648eaa91f2d5a006510ca7c28d49c5ab652969e5 Mon Sep 17 00:00:00 2001 From: Kate Date: Wed, 3 Jul 2024 12:40:44 +0100 Subject: [PATCH 2/2] Add every listed checksums of a given archive to the cache when fetching from it --- master_changes.md | 3 ++ src/client/opamAdminCommand.ml | 11 +++--- src/repository/opamRepository.ml | 47 ++++++++++++---------- src/repository/opamRepository.mli | 3 +- tests/reftests/admin-cache.test | 65 +++++++++++++++---------------- tests/reftests/archive.test | 6 +-- tests/reftests/extrasource.test | 12 +++--- 7 files changed, 78 insertions(+), 69 deletions(-) diff --git a/master_changes.md b/master_changes.md index 27aad71dce4..c5a62c10f8e 100644 --- a/master_changes.md +++ b/master_changes.md @@ -29,6 +29,7 @@ users) ## Install * Fix package name display for no agreement conflicts [#6055 @rjbou - fix #6030] + * Make fetching an archive from cache add missing symlinks [#6068 @kit-ty-kate - fix #6064] ## Remove @@ -95,6 +96,7 @@ users) ## Admin * Change hash cache location from `~/.cache` to `/download-cache/hash-cache` [#6103 @rjbou] + * Make `opam admin cache` add missing symlinks [#6068 @kit-ty-kate - fix #6064] ## Opam installer @@ -148,6 +150,7 @@ users) ## opam-client ## opam-repository + * `OpamRepository.fetch_from_cache`: when an archive is found, add a symlink (or copy) for the ones found in opam file but not in cache [#6068 @kit-ty-kate] ## opam-state * `OpamStateConfig.opamroot_with_provenance`: restore previous behaviour to `OpamStateConfig.opamroot` for compatibility with third party code [#6047 @dra27] diff --git a/src/client/opamAdminCommand.ml b/src/client/opamAdminCommand.ml index 54543adf939..9714372eed6 100644 --- a/src/client/opamAdminCommand.ml +++ b/src/client/opamAdminCommand.ml @@ -162,12 +162,13 @@ let package_files_to_cache repo_root cache_dir cache_urls OpamConsole.warning "[%s] no checksum, not caching" (OpamConsole.colorise `green label); Done errors - | best_chks :: _ -> - let cache_file = - OpamRepository.cache_file cache_dir best_chks + | _::_ -> + let cache_files = + List.map (OpamRepository.cache_file cache_dir) checksums in let error_opt = - if not recheck && OpamFilename.exists cache_file then Done None + if not recheck && List.for_all OpamFilename.exists cache_files then + Done None else OpamRepository.pull_file_to_cache label ~cache_urls ~cache_dir @@ -190,7 +191,7 @@ let package_files_to_cache repo_root cache_dir cache_urls let link = OpamFilename.Op.(link_dir / OpamPackage.to_string nv // name) in - OpamFilename.link ~relative:true ~target:cache_file ~link) + OpamFilename.link ~relative:true ~target:(List.hd cache_files) ~link) link; errors in diff --git a/src/repository/opamRepository.ml b/src/repository/opamRepository.ml index 826257a14bc..f321669e707 100644 --- a/src/repository/opamRepository.ml +++ b/src/repository/opamRepository.ml @@ -44,6 +44,11 @@ let cache_file cache_dir checksum = in aux cache_dir (OpamHash.to_path checksum) +let link_files ~target f l = + List.iter (fun x -> + OpamFilename.link ~relative:true ~target ~link:(f x)) + l + let fetch_from_cache = let currently_downloading = ref [] in let rec no_concurrent_dls key f x = @@ -93,21 +98,27 @@ let fetch_from_cache = failwith "Version control not allowed as cache URL" in try - let hit_file = - OpamStd.List.find_map (fun ck -> + match + List.fold_left (fun (hit, misses) ck -> let f = cache_file cache_dir ck in - if OpamFilename.exists f then Some f else None) - checksums - in - if List.for_all - (fun ck -> OpamHash.check_file (OpamFilename.to_string hit_file) ck) - checksums - then Done (Up_to_date (hit_file, OpamUrl.empty)) - else mismatch hit_file + if OpamFilename.exists f + then (if hit = None then (Some f, misses) else (hit, misses)) + else (hit, f :: misses)) + (None, []) checksums + with + | None, _ -> raise Not_found + | Some hit_file, miss_files -> + if List.for_all + (fun ck -> OpamHash.check_file (OpamFilename.to_string hit_file) ck) + checksums + then begin + link_files ~target:hit_file Fun.id miss_files; + Done (Up_to_date (hit_file, OpamUrl.empty)) + end else + mismatch hit_file with Not_found -> match checksums with | [] -> let m = "cache miss" in Done (Not_available (Some m, m)) - | checksum::_ -> - (* Try all cache urls in order, but only the first checksum *) + | checksum::other_checksums -> let local_file = cache_file cache_dir checksum in let tmpfile = OpamFilename.add_extension local_file "tmp" in let rec try_cache_dl = function @@ -125,6 +136,7 @@ let fetch_from_cache = checksums then (OpamFilename.move ~src:tmpfile ~dst:local_file; + link_files ~target:local_file (cache_file cache_dir) other_checksums; Done (Result (local_file, root_cache_url))) else mismatch tmpfile in @@ -151,14 +163,9 @@ let validate_and_add_to_cache label url cache_dir file checksums = (let checksums = OpamHash.sort checksums in match cache_dir, checksums with | Some dir, best_chks :: others_chks -> - OpamFilename.copy ~src:file ~dst:(cache_file dir best_chks); - List.iter (fun checksum -> - let target = cache_file dir best_chks in - let link = cache_file dir checksum in - try - OpamFilename.link ~relative:true ~target ~link - with Sys_error _ -> ()) - others_chks; + let target = cache_file dir best_chks in + OpamFilename.copy ~src:file ~dst:target; + link_files ~target (cache_file dir) others_chks; | _ -> ()); true diff --git a/src/repository/opamRepository.mli b/src/repository/opamRepository.mli index ed1f372e1f3..f66a3a73c2e 100644 --- a/src/repository/opamRepository.mli +++ b/src/repository/opamRepository.mli @@ -58,7 +58,8 @@ val pull_file: unit download OpamProcess.job (** Same as [pull_file], but without a destination file: just ensures the file - is present in the cache. *) + is present in the cache and that every listed checksums are correctly linked + to the archive, otherwise it adds the missing links. *) val pull_file_to_cache: string -> cache_dir:dirname -> ?cache_urls:url list -> OpamHash.t list -> url list -> string download OpamProcess.job diff --git a/tests/reftests/admin-cache.test b/tests/reftests/admin-cache.test index b87ee3acd1f..af344519b90 100644 --- a/tests/reftests/admin-cache.test +++ b/tests/reftests/admin-cache.test @@ -147,40 +147,44 @@ amet : sha512 : exists ### # in cache, the archiv is the strongest one ### rm -rf cache/md5 cache/sha256 ### opam admin cache | unordered +[dolor.1] synchronised (cached) +[ipsum.1] synchronised (cached) +[sit.1] synchronised (cached) +[amet.1] synchronised (cached) Done. ### sh cache-analysi.sh -ipsum : md5 : not found -ipsum : sha256 : not found +ipsum : md5 : exists, link to sha512 +ipsum : sha256 : exists, link to sha512 ipsum : sha512 : exists -- -dolor : md5 : not found -dolor : sha256 : not found +dolor : md5 : exists, link to sha512 +dolor : sha256 : exists, link to sha512 dolor : sha512 : exists -- -sit : md5 : not found -sit : sha256 : not found +sit : md5 : exists, link to sha512 +sit : sha256 : exists, link to sha512 sit : sha512 : exists -- -amet : md5 : not found -amet : sha256 : not found +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 amet : sha512 : exists ### opam admin cache | unordered Done. ### sh cache-analysi.sh -ipsum : md5 : not found -ipsum : sha256 : not found +ipsum : md5 : exists, link to sha512 +ipsum : sha256 : exists, link to sha512 ipsum : sha512 : exists -- -dolor : md5 : not found -dolor : sha256 : not found +dolor : md5 : exists, link to sha512 +dolor : sha256 : exists, link to sha512 dolor : sha512 : exists -- -sit : md5 : not found -sit : sha256 : not found +sit : md5 : exists, link to sha512 +sit : sha256 : exists, link to sha512 sit : sha512 : exists -- -amet : md5 : not found -amet : sha256 : not found +amet : md5 : exists, link to sha512 +amet : sha256 : exists, link to sha512 amet : sha512 : exists ### ::::::::::::::::::::::::::::::::::::::::::: ### :III: remove strongest hash and recompute : @@ -266,10 +270,10 @@ mv "$file.tmp" "$file" ### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 url.src: file://${BASEDIR}/arch-ipsum.tgz url.checksum: md5=IP_MD5 -### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256 | "${DL_SHA512}" -> DL_SHA512 +### opam show --just-file ./packages/dolor/dolor.1/opam --field url.src,url.checksum | "${DL_MD5}" -> DL_MD5 | "${DL_SHA256}" -> DL_SHA256 url.src: file://${BASEDIR}/arch-dolor.tgz url.checksum: md5=DL_MD5, sha256=DL_SHA256 -### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_MD5}" -> ST_MD5 | "${ST_SHA256}" -> ST_SHA256 | "${ST_SHA512}" -> ST_SHA512 +### opam show --just-file ./packages/sit/sit.1/opam --field url.src,url.checksum | "${ST_SHA256}" -> ST_SHA256 url.src: file://${BASEDIR}/arch-sit.tgz url.checksum: sha256=ST_SHA256 ### rm -rf cache @@ -299,7 +303,7 @@ amet : sha512 : exists ### opam admin add-hashes sha512 --package ipsum ### opam admin add-hashes sha512 --package dolor ### opam admin add-hashes md5 --package sit -[file://${BASEDIR}/arch-sit.tgz] downloaded from file://${BASEDIR}/cache +[file://${BASEDIR}/arch-sit.tgz] found in cache ### opam admin add-hashes sha512 --package sit ### opam show --just-file ./packages/ipsum/ipsum.1/opam --field url.src,url.checksum | "${IP_MD5}" -> IP_MD5 | "${IP_SHA512}" -> IP_SHA512 url.src: file://${BASEDIR}/arch-ipsum.tgz @@ -335,15 +339,15 @@ Done. ### sh cache-analysi.sh ipsum : md5 : exists ipsum : sha256 : not found -ipsum : sha512 : not found +ipsum : sha512 : exists, link to md5 -- dolor : md5 : exists, link to sha256 dolor : sha256 : exists -dolor : sha512 : not found +dolor : sha512 : exists, link to sha256 -- -sit : md5 : not found +sit : md5 : exists, link to sha256 sit : sha256 : exists -sit : sha512 : not found +sit : sha512 : exists, link to sha256 -- amet : md5 : exists, link to sha512 amet : sha256 : exists, link to sha512 @@ -369,12 +373,10 @@ opam-version: "2.0" MD5(arch-evil.tgz)= LR_MD5 ### sh add-urls.sh lorem ### opam admin cache | unordered -[dolor.1] synchronised (cached) -[ipsum.1] synchronised (cached) -[sit.1] synchronised (cached) [lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz) Done. ### opam admin add-hashes sha256 --package lorem +[file://${BASEDIR}/arch-lorem.tgz] downloaded from file://${BASEDIR}/cache ### sh analysi.sh lorem md5 $LR_MD5 sh analysi.sh lorem sha256 $LR_SHA256 @@ -385,19 +387,17 @@ lorem : sha256 : not found lorem : sha512 : not found ### opam admin cache | unordered [lorem.1] synchronised (cached) -[dolor.1] synchronised (cached) -[ipsum.1] synchronised (cached) -[sit.1] synchronised (cached) Done. ### sh cache-analysi.sh lorem : md5 : exists -lorem : sha256 : not found +lorem : sha256 : exists, link to md5 lorem : sha512 : not found ### mv arch-evil.tgz cache/md5/$PRE_LR_MD5/$LR_MD5 ### opam admin add-hashes sha512 --package lorem +[file://${BASEDIR}/arch-lorem.tgz] found in cache ### sh cache-analysi.sh lorem : md5 : exists -lorem : sha256 : not found +lorem : sha256 : exists, link to md5 lorem : sha512 : not found ### opam admin cache | "${LR_MD5}" -> LR_MD5 | "${LR_SHA256}" -> LR_SHA256 | "${LR_SHA512}" -> LR_SHA512 [ERROR] Conflicting file hashes, or broken or compromised cache! @@ -405,9 +405,6 @@ lorem : sha512 : not found - sha256=LR_SHA256 (MISMATCH) - md5=LR_MD5 (match) -[dolor.1] synchronised (cached) -[ipsum.1] synchronised (cached) -[sit.1] synchronised (cached) [lorem.1] synchronised (file://${BASEDIR}/arch-lorem.tgz) Done. ### sh cache-analysi.sh diff --git a/tests/reftests/archive.test b/tests/reftests/archive.test index 69c2eaac0e3..aa6fcc10680 100644 --- a/tests/reftests/archive.test +++ b/tests/reftests/archive.test @@ -354,7 +354,7 @@ The following actions will be performed: Done. ### sh check-cache.sh MD5: archive, with matching checksum -SHA256: not found +SHA256: link, to md5 archive ### opam remove good-md5-good-sha256 The following actions will be performed: === remove 1 package @@ -405,7 +405,7 @@ The following actions will be performed: Done. ### sh check-cache.sh MD5: archive, with matching checksum -SHA256: not found +SHA256: link, to md5 archive ### opam remove good-sha256-good-md5 The following actions will be performed: === remove 1 package @@ -962,7 +962,7 @@ The following actions will be performed: -> installed good-sha256-good-md5.1 Done. ### sh check-cache.sh -MD5: not found +MD5: link, to sha256 archive SHA256: archive, with matching checksum ### opam clean --download-cache Clearing cache of downloaded files diff --git a/tests/reftests/extrasource.test b/tests/reftests/extrasource.test index f46207d1e0c..fa65a343d65 100644 --- a/tests/reftests/extrasource.test +++ b/tests/reftests/extrasource.test @@ -429,7 +429,7 @@ The following actions will be performed: Done. ### sh check-cache.sh MD5: patch, with matching checksum -SHA256: not found +SHA256: link, to md5 patch ### opam remove good-md5-good-sha256 The following actions will be performed: === remove 1 package @@ -480,7 +480,7 @@ The following actions will be performed: Done. ### sh check-cache.sh MD5: patch, with matching checksum -SHA256: not found +SHA256: link, to md5 patch ### opam remove good-sha256-good-md5 The following actions will be performed: === remove 1 package @@ -1090,7 +1090,7 @@ The following actions will be performed: -> installed good-sha256-good-md5.1 Done. ### sh check-cache.sh -MD5: not found +MD5: link, to sha256 patch SHA256: patch, with matching checksum ### opam clean --download-cache Clearing cache of downloaded files @@ -1120,11 +1120,11 @@ The following actions will be performed: Done. ### sh check-cache.sh MD5: patch, with matching checksum -SHA256: not found +SHA256: link, to md5 patch ### :III:c: corrupt md5 patch in cache ### sh check-cache.sh MD5: patch, with matching checksum -SHA256: not found +SHA256: link, to md5 patch ### opam remove good-sha256-good-md5 The following actions will be performed: === remove 1 package @@ -1136,7 +1136,7 @@ Done. ### cp check-cache.sh "$PATCH_MD5_PATH" ### sh check-cache.sh MD5: patch, with mismatching checksum -SHA256: not found +SHA256: link, to md5 patch ### opam install good-sha256-good-md5 | '[0-9a-z]{32,64}' -> 'hash' The following actions will be performed: === install 1 package