From a72ae316c31c10d94d0a9a340b43fc240d45deeb Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Tue, 2 Aug 2016 22:20:15 -0700 Subject: [PATCH 1/6] Added caching ability. --- src/lib/abs.sh.in | 14 +++++++++++--- src/lib/pkgbuild.sh.in | 4 +++- src/lib/util.sh.in | 1 + src/man/yaourt.8 | 5 +++++ src/yaourt.sh.in | 8 +++++--- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/lib/abs.sh.in b/src/lib/abs.sh.in index 77cf57f..452662a 100644 --- a/src/lib/abs.sh.in +++ b/src/lib/abs.sh.in @@ -395,30 +395,38 @@ sync_packages() { fi [[ $1 ]] || return 0 # Install from arguments - declare -A pkgs_search pkgs_found + declare -A pkgs_search pkgs_found pkgs_cached declare -a repo_pkgs aur_pkgs bin_pkgs local _arg=("$@") _pkg repo pkg target [[ $SP_ARG ]] && _arg=($(pacman_parse -T "${_arg[@]}")) for _pkg in "${_arg[@]}"; do pkgs_search[$_pkg]=1; done # Search for exact match, pkg which provides it, then in AUR - while read repo pkg target; do + while read repo pkg target version; do ((pkgs_search[$target])) || continue unset pkgs_search[$target] ((pkgs_found[$pkg])) && continue pkgs_found[$pkg]=1 + [[ $CACHE_DIR ]] && + [[ "$(ls "$CACHE_DIR" | grep "^${pkg}-${version}")" ]] && + pkgs_cached["${pkg}-${version}"]=1 if [[ "${repo}" != "aur" ]]; then repo_pkgs+=("${repo}/${pkg}") else ((NEEDED)) && pkgquery -Qqii "$target" && continue aur_pkgs+=("$pkg") fi - done < <(pkgquery -f "%r %n %t" -1SAii "${!pkgs_search[@]}") + done < <(pkgquery -f "%r %n %t %v" -1SAii "${!pkgs_search[@]}") + # In case there is a double version check, eg. pkg1>0.1 pkg2<0.3 # package-query does only one check, so perform the remaining checks for pkg in "${!pkgs_search[@]}"; do _pkg=${pkg%%[<>=]*} + _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" if [[ " ${!pkgs_found[@]} " =~ " $_pkg " ]]; then pkgquery -q -1SAii "$pkg" && unset pkgs_search[$pkg] + elif [[ $_cached ]]; then + su_pacman -U "${PACMAN_S_ARG[@]}" "$CACHE_DIR/$_cached" && + unset pkgs_search[$pkg] || return 1 fi done bin_pkgs=("${!pkgs_search[@]}") diff --git a/src/lib/pkgbuild.sh.in b/src/lib/pkgbuild.sh.in index a3a8489..7d4984b 100644 --- a/src/lib/pkgbuild.sh.in +++ b/src/lib/pkgbuild.sh.in @@ -403,11 +403,13 @@ package_loop() { *) return 99 ;; # should never execute esac done - (( ! ret )) && (( ! failed )) && + (( ! ret )) && (( ! failed )) && (( ! DOWNLOAD_ONLY )) && ( [[ $MAJOR == "pkgbuild" ]] && (( INSTALL )) || ! [[ $MAJOR == "pkgbuild" ]] ) && { install_package || failed=1; } [[ $MAJOR == "pkgbuild" ]] && (( ! INSTALL )) && cp -v "$YPKGDEST/"*.pkg.* $YPKGBUILDDIR + (( DOWNLOAD_ONLY )) && cp -uv "$YPKGDEST/"*.pkg.* "$CACHE_DIR" || failed=1 + rm -r "$YPKGDEST" return $failed } diff --git a/src/lib/util.sh.in b/src/lib/util.sh.in index 486a8c6..4629828 100644 --- a/src/lib/util.sh.in +++ b/src/lib/util.sh.in @@ -168,6 +168,7 @@ init_paths() { YAOURTTMPDIR="$TMPDIR/yaourt-tmp-$(id -un)" mkdir -p "$YAOURTTMPDIR" || check_dir YAOURTTMPDIR || die 1 (( EXPORT )) && [[ $EXPORTDIR ]] && { check_dir EXPORTDIR || die 1; } + [[ "$CACHE_DIR" ]] && { check_dir CACHE_DIR || die 1; } parse_pacman_conf } diff --git a/src/man/yaourt.8 b/src/man/yaourt.8 index d9d833f..51b22e1 100644 --- a/src/man/yaourt.8 +++ b/src/man/yaourt.8 @@ -295,6 +295,11 @@ option, put them in quotes. .RS 4 Upgrade all packages that are out of date\&. .RE +.PP +\fB\-w, \-\-download-only \fR +.RS 4 +Download and compile packages, but do not install. Place output in \&. +.RE .SH "HANDLING CONFIG FILES" .sp With AUTOSAVEBACKUPFILE (\fByaourtrc\fR(5)), yaourt will search and save all files marked as backup\&. These files can be used later with \fIyaourt \-C\fR to automerge current configuration files with new ones\&. diff --git a/src/yaourt.sh.in b/src/yaourt.sh.in index c07c25f..2c6d405 100755 --- a/src/yaourt.sh.in +++ b/src/yaourt.sh.in @@ -297,7 +297,7 @@ unset MAJOR AUR SEARCH BUILD REFRESH SYSUPGRADE CLEAN \ IGNOREGRP IGNOREPKG CHANGELOG LIST INFO RSORT SORT UNREQUIRED \ FOREIGN GROUP QUERYTYPE QUIET DEPENDS PRINT \ AURVOTEINSTALLED CUSTOMIZEPKGINSTALLED PACMAN_CMD \ - NEEDED + NEEDED CACHE_DIR DOWNLOAD_ONLY_DIR # Disable color if output is not a terminal [[ -t 1 ]] || { ((USECOLOR!=2)) && USECOLOR=0; USEPAGER=0; TERMINALTITLE=0; } @@ -358,14 +358,12 @@ while [[ $1 ]]; do -u|--upgrades) (( UPGRADES ++ )); program_arg $A_PQ $1;; -V|--version) version; exit 0;; -v|--verbose) program_arg $((A_PQ | A_PS)) $1;; - -w|--downloadonly) PACMAN_CMD=1;; -y|--refresh) (( REFRESH ++ ));; --asdeps) program_arg $A_PS $1;; --asexplicit) program_arg $A_PS $1;; --assume-installed) program_arg $A_PS $1 "$2"; shift;; --arch) program_arg $A_PC $1 "$2"; shift;; - --cachedir) program_arg $A_PC $1 "$2"; shift;; --changelog) [[ ! $PACMAN_CMD ]] && PACMAN_CMD=0;; --clean) ((CLEAN++));; --color) USECOLOR=2;; @@ -399,10 +397,12 @@ while [[ $1 ]]; do -) break;; # support smth like echo package | yaourt -S - -a|--aur) AUR=1; AURUPGRADE=1; AURSEARCH=1;; -b|--build) ((BUILD++));; + -w|--downloadonly) DOWNLOAD_ONLY="1"; CACHE_DIR="$2"; shift;; --) shift; ARGS+=("$@"); break;; --aur-url) AURURL=$2; shift;; --backupfile) BACKUPFILE=$2; shift;; + --cachedir) CACHE_DIR="$2"; shift;; --conflicts) QUERYTYPE=${1:2};; --date) SORT=1;; --depends) QUERYTYPE=${1:2};; @@ -435,6 +435,8 @@ init_paths init_color [[ $PACMAN_CMD ]] && pacman_cmd $PACMAN_CMD +[[ $CACHE_DIR ]] && program_arg $A_PC --cachedir "$CACHE_DIR" + # No options [[ -z $MAJOR ]] && yaourt_no_action # Complete options From 405ea36b91268596371a6534938dde2ea120b396 Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Mon, 8 Aug 2016 20:18:39 -0700 Subject: [PATCH 2/6] More caching improvements. --- src/Makefile | 2 +- src/lib/abs.sh.in | 52 ++++++++++++++++++++++++++++-------------- src/lib/pkgbuild.sh.in | 5 +++- src/lib/util.sh.in | 4 +++- src/yaourt.sh.in | 2 +- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/Makefile b/src/Makefile index cca8e84..cd9baa5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -2,7 +2,7 @@ VERSION = $(shell git describe --abbrev=4) DESTDIR = -PREFIX = /usr/local +PREFIX = /home/aoneill/yaourt/build SHELL = /bin/bash INSTALL = /usr/bin/env install MSGFMT = /usr/bin/msgfmt diff --git a/src/lib/abs.sh.in b/src/lib/abs.sh.in index 452662a..f650829 100644 --- a/src/lib/abs.sh.in +++ b/src/lib/abs.sh.in @@ -400,35 +400,53 @@ sync_packages() { local _arg=("$@") _pkg repo pkg target [[ $SP_ARG ]] && _arg=($(pacman_parse -T "${_arg[@]}")) for _pkg in "${_arg[@]}"; do pkgs_search[$_pkg]=1; done + # Search for exact match, pkg which provides it, then in AUR - while read repo pkg target version; do - ((pkgs_search[$target])) || continue - unset pkgs_search[$target] - ((pkgs_found[$pkg])) && continue - pkgs_found[$pkg]=1 - [[ $CACHE_DIR ]] && - [[ "$(ls "$CACHE_DIR" | grep "^${pkg}-${version}")" ]] && - pkgs_cached["${pkg}-${version}"]=1 - if [[ "${repo}" != "aur" ]]; then - repo_pkgs+=("${repo}/${pkg}") - else - ((NEEDED)) && pkgquery -Qqii "$target" && continue - aur_pkgs+=("$pkg") - fi - done < <(pkgquery -f "%r %n %t %v" -1SAii "${!pkgs_search[@]}") + local tmpfile=$(mktemp --tmpdir="$YAOURTTMPDIR") + + pkgquery -f "%r %n %t %v" -1SAii "${!pkgs_search[@]}" 2>/dev/null >$tmpfile + [[ "$?" != "0" ]] && warning "no network connection" || \ + while read repo pkg target version; do + ((pkgs_search[$target])) || continue + unset pkgs_search[$target] + ((pkgs_found[$pkg])) && continue + pkgs_found[$pkg]=1 + [[ "$(ls "$CACHE_DIR" | grep "^${pkg}")" ]] && \ + pkgs_cached["${pkg}"]="${version}" + if [[ "${repo}" != "aur" ]]; then + repo_pkgs+=("${repo}/${pkg}") + else + ((NEEDED)) && pkgquery -Qqii "$target" && continue + aur_pkgs+=("$pkg") + fi + done < $tmpfile # In case there is a double version check, eg. pkg1>0.1 pkg2<0.3 # package-query does only one check, so perform the remaining checks + cached_pkgs=() for pkg in "${!pkgs_search[@]}"; do _pkg=${pkg%%[<>=]*} _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" if [[ " ${!pkgs_found[@]} " =~ " $_pkg " ]]; then pkgquery -q -1SAii "$pkg" && unset pkgs_search[$pkg] elif [[ $_cached ]]; then - su_pacman -U "${PACMAN_S_ARG[@]}" "$CACHE_DIR/$_cached" && - unset pkgs_search[$pkg] || return 1 + cached_pkgs+=("$CACHE_DIR/$_cached") && unset pkgs_search[$pkg] fi done + + # Install from cached packages + for _pkg in "${!pkgs_cached[@]}"; do + _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" + if [[ $_cached ]]; then + [[ "$_cached" = ${_pkg}-${pkgs_cached["$_pkg"]}* ]] && + cached_pkgs+=("$CACHE_DIR/$_cached") + fi + done + (( ! DOWNLOAD_ONLY)) && for _pkg in "${cached_pkgs[@]}"; do + su_pacman -U "${PACMAN_S_ARG[@]}" "$_pkg" || \ + manage_error "$_pkg" || return 1 + done + bin_pkgs=("${!pkgs_search[@]}") for _pkg in "${repo_pkgs[@]}"; do install_from_abs "$_pkg" || return 1 diff --git a/src/lib/pkgbuild.sh.in b/src/lib/pkgbuild.sh.in index 7d4984b..000c328 100644 --- a/src/lib/pkgbuild.sh.in +++ b/src/lib/pkgbuild.sh.in @@ -282,6 +282,10 @@ build_package() { # Build SRCDEST="$YSRCDEST" PKGDEST="$YPKGDEST" y_makepkg -s -c -f -p ./PKGBUILD + # Cache it + msg $(_gettext 'Caching locally: %s %s-%s' "$pkgbase" "$pkgver" "$pkgrel") + cp -uv "$YPKGDEST/"*.pkg.* "$CACHE_DIR" || failed=1 + if (( $? )); then error $(_gettext 'Makepkg was unable to build %s.' "$pkgbase") return 1 @@ -408,7 +412,6 @@ package_loop() { { install_package || failed=1; } [[ $MAJOR == "pkgbuild" ]] && (( ! INSTALL )) && cp -v "$YPKGDEST/"*.pkg.* $YPKGBUILDDIR - (( DOWNLOAD_ONLY )) && cp -uv "$YPKGDEST/"*.pkg.* "$CACHE_DIR" || failed=1 rm -r "$YPKGDEST" return $failed diff --git a/src/lib/util.sh.in b/src/lib/util.sh.in index 4629828..3d4e145 100644 --- a/src/lib/util.sh.in +++ b/src/lib/util.sh.in @@ -168,7 +168,9 @@ init_paths() { YAOURTTMPDIR="$TMPDIR/yaourt-tmp-$(id -un)" mkdir -p "$YAOURTTMPDIR" || check_dir YAOURTTMPDIR || die 1 (( EXPORT )) && [[ $EXPORTDIR ]] && { check_dir EXPORTDIR || die 1; } - [[ "$CACHE_DIR" ]] && { check_dir CACHE_DIR || die 1; } + [[ ! "$CACHE_DIR" ]] && CACHE_DIR="@cachedir@" && + ! [[ -d "$CACHE_DIR" ]] && { mkdir -p "$CACHE_DIR" || die 1; } + check_dir CACHE_DIR || die 1; parse_pacman_conf } diff --git a/src/yaourt.sh.in b/src/yaourt.sh.in index 2c6d405..1a7d9c1 100755 --- a/src/yaourt.sh.in +++ b/src/yaourt.sh.in @@ -397,7 +397,7 @@ while [[ $1 ]]; do -) break;; # support smth like echo package | yaourt -S - -a|--aur) AUR=1; AURUPGRADE=1; AURSEARCH=1;; -b|--build) ((BUILD++));; - -w|--downloadonly) DOWNLOAD_ONLY="1"; CACHE_DIR="$2"; shift;; + -w|--downloadonly) DOWNLOAD_ONLY="1";; --) shift; ARGS+=("$@"); break;; --aur-url) AURURL=$2; shift;; From b42b78870e7beb1d57ac556b01bcae13e62b9eb2 Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Mon, 8 Aug 2016 20:19:37 -0700 Subject: [PATCH 3/6] Reverted issues with Makefile --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index cd9baa5..cca8e84 100644 --- a/src/Makefile +++ b/src/Makefile @@ -2,7 +2,7 @@ VERSION = $(shell git describe --abbrev=4) DESTDIR = -PREFIX = /home/aoneill/yaourt/build +PREFIX = /usr/local SHELL = /bin/bash INSTALL = /usr/bin/env install MSGFMT = /usr/bin/msgfmt From 52be3b14ff4e78897d0f418d50fa596113aec75b Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Tue, 9 Aug 2016 17:22:04 -0700 Subject: [PATCH 4/6] Fixed various edge cases with cache logic. --- src/lib/abs.sh.in | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/lib/abs.sh.in b/src/lib/abs.sh.in index f650829..3d871bb 100644 --- a/src/lib/abs.sh.in +++ b/src/lib/abs.sh.in @@ -395,8 +395,8 @@ sync_packages() { fi [[ $1 ]] || return 0 # Install from arguments - declare -A pkgs_search pkgs_found pkgs_cached - declare -a repo_pkgs aur_pkgs bin_pkgs + declare -A pkgs_search pkgs_found pkgs_cached repo_pkgs aur_pkgs cached_pkgs + declare -a bin_pkgs local _arg=("$@") _pkg repo pkg target [[ $SP_ARG ]] && _arg=($(pacman_parse -T "${_arg[@]}")) for _pkg in "${_arg[@]}"; do pkgs_search[$_pkg]=1; done @@ -405,45 +405,46 @@ sync_packages() { local tmpfile=$(mktemp --tmpdir="$YAOURTTMPDIR") pkgquery -f "%r %n %t %v" -1SAii "${!pkgs_search[@]}" 2>/dev/null >$tmpfile - [[ "$?" != "0" ]] && warning "no network connection" || \ + [[ "$?" != "0" ]] && warning "no network connection" || while read repo pkg target version; do ((pkgs_search[$target])) || continue unset pkgs_search[$target] ((pkgs_found[$pkg])) && continue pkgs_found[$pkg]=1 - [[ "$(ls "$CACHE_DIR" | grep "^${pkg}")" ]] && \ + [[ "$(ls "$CACHE_DIR" | grep "^${pkg}")" ]] && pkgs_cached["${pkg}"]="${version}" if [[ "${repo}" != "aur" ]]; then - repo_pkgs+=("${repo}/${pkg}") + repo_pkgs[$pkg]="${repo}/${pkg}" else ((NEEDED)) && pkgquery -Qqii "$target" && continue - aur_pkgs+=("$pkg") + aur_pkgs[$pkg]=1 fi done < $tmpfile # In case there is a double version check, eg. pkg1>0.1 pkg2<0.3 # package-query does only one check, so perform the remaining checks - cached_pkgs=() for pkg in "${!pkgs_search[@]}"; do - _pkg=${pkg%%[<>=]*} - _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" + local _pkg=${pkg%%[<>=]*} + local _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" if [[ " ${!pkgs_found[@]} " =~ " $_pkg " ]]; then pkgquery -q -1SAii "$pkg" && unset pkgs_search[$pkg] elif [[ $_cached ]]; then - cached_pkgs+=("$CACHE_DIR/$_cached") && unset pkgs_search[$pkg] + cached_pkgs[$_pkg]="$CACHE_DIR/$_cached" && unset pkgs_search[$pkg] fi done # Install from cached packages for _pkg in "${!pkgs_cached[@]}"; do - _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" + local _cached="$(ls "$CACHE_DIR" | grep "^$_pkg")" if [[ $_cached ]]; then [[ "$_cached" = ${_pkg}-${pkgs_cached["$_pkg"]}* ]] && - cached_pkgs+=("$CACHE_DIR/$_cached") + cached_pkgs[$_pkg]="$CACHE_DIR/$_cached" fi done - (( ! DOWNLOAD_ONLY)) && for _pkg in "${cached_pkgs[@]}"; do - su_pacman -U "${PACMAN_S_ARG[@]}" "$_pkg" || \ + (( ! DOWNLOAD_ONLY)) && for _pkg in "${!cached_pkgs[@]}"; do + local _file="${cached_pkgs[$_pkg]}" + su_pacman -U "${PACMAN_S_ARG[@]}" "$_file" && + unset aur_pkgs[$_pkg] && unset repo_pkgs[$_pkg] || manage_error "$_pkg" || return 1 done @@ -458,7 +459,7 @@ sync_packages() { su_pacman -S "${PACMAN_S_ARG[@]}" "${bin_pkgs[@]}" || return 1 fi fi - for _pkg in "${aur_pkgs[@]}"; do + for _pkg in "${!aur_pkgs[@]}"; do install_from_aur "$_pkg" || return 1 done return 0 From 8e0796035976c9a1f34b52780edd82c6e1c05495 Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Tue, 9 Aug 2016 21:24:14 -0700 Subject: [PATCH 5/6] Added section on --cachedir to man page. --- src/man/yaourt.8 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/man/yaourt.8 b/src/man/yaourt.8 index 51b22e1..b381416 100644 --- a/src/man/yaourt.8 +++ b/src/man/yaourt.8 @@ -262,6 +262,11 @@ Specify a custom AUR url (default to https://aur.archlinux.org)\&. Build from sources, ABS for official packages, or AUR if packages is not found\&. Specify this option twice to build all dependencies\&. .RE .PP +\fB\-\-cachedir \fR +.RS 4 +Use as a local cache for \fByaourt\fR. +.RE +.PP \fB\-\-devel\fR .RS 4 With @@ -296,9 +301,9 @@ option, put them in quotes. Upgrade all packages that are out of date\&. .RE .PP -\fB\-w, \-\-download-only \fR +\fB\-w, \-\-download-only\fR .RS 4 -Download and compile packages, but do not install. Place output in \&. +Download and compile packages, but do not install. Place output in . See \fI--cachedir\fR\&. .RE .SH "HANDLING CONFIG FILES" .sp From 4c48675ae5f84553d47ac5fdb245f21b5274444f Mon Sep 17 00:00:00 2001 From: Alex O'Neill Date: Tue, 9 Aug 2016 21:44:59 -0700 Subject: [PATCH 6/6] Consistency changes to be more inline with pacman. --- src/man/yaourt.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/man/yaourt.8 b/src/man/yaourt.8 index b381416..61ce300 100644 --- a/src/man/yaourt.8 +++ b/src/man/yaourt.8 @@ -264,7 +264,7 @@ Build from sources, ABS for official packages, or AUR if packages is not found\& .PP \fB\-\-cachedir \fR .RS 4 -Use as a local cache for \fByaourt\fR. +Use as a local cache for \fByaourt\fR, rather than the standard location, /var/cache/pacman/pkg. .RE .PP \fB\-\-devel\fR