mirrorcheck

#!/bin/bash
#
# Check that a mirror repo has the same package and db files as the given local folder.
#

printf2() { printf "$@" >&2 ; }
echo2()   { echo  "$@" >&2 ; }
echo2i()  { echoi "$@" >&2 ; }
echoi() {
    local level=""
    local xx
    local optcount=0
    for xx in "$@" ; do
        case "$xx" in
            -l=*) level=${xx:3} ; ((optcount++)) ;;
            -l*)  level=${xx:2} ; ((optcount++)) ;;
            -*) DIE "unsupported option $xx" ;;        # non-options cannot start with '-'
            *) break ;;                                # all options must be before other params
        esac
    done
    if [ -n "$level" ] ; then
        shift $optcount
    else
        level=1
    fi
    
    printf "%*s" $((level*4)) ""
    echo2 "$@"
}

DIE() {
    echo2 "Error: $1"
    Usage
    exit 1
}

NameCheck() {
    local xx yy
    for xx in "$@" ; do
        for yy in $remotefilelist ; do
            yy="${yy/\%2B/+}"                      # convert character code '%2B' to '+'
            test "$xx" = "$yy" && break
        done
        if [ "$xx" != "$yy" ] ; then
            echo2i "FAIL: $xx is missing from $reponame !"
            problemfiles+="$xx "
        else
            case "$mode" in
                fastall) ;;
                *) case $only_fails in
                       no) echo2 "    OK: $xx" ;;
                       yes) printf2 "\r%s" "$((++handled))/$count" ;;
                   esac
                   ;;
            esac
        fi
    done
}

ContentsCheck() {
    local xx
    local url=""
    local repo_dir=""
    local orig_url="${mirror_url%/}"

    for xx in "$@" ; do
        case "$xx" in
            state)
                url="${orig_url}/../.."
                repo_dir="../../repo"
                ;;
            *)
                url="${orig_url}"
                repo_dir="."
                ;;
        esac

        if ! curl -Lfsm 30 -o "$tmpdir/$xx" "$url/$xx" ; then
            echo2i "FAIL: $xx fetching failed."
            problemfiles+="$xx "
            continue
        fi
        if ! cmp "$repo_dir/$xx" "$tmpdir/$xx" >/dev/null ; then
            echo2i "FAIL: $xx is different !"
            problemfiles+="$xx "
            continue
        fi
        rm -f "$tmpdir/$xx"
        case $only_fails in
            no)  echo2 "    OK: $xx" ;;
            yes) printf2 "\r%s" "$((++handled))/$count" ;;
        esac
    done
}

OneMirror() {
    local remotefilelist
    local packages repofiles
    local problemfiles=""
    local mirror_url
    local repo=endeavouros

    mirror_url="${MIRROR_URLS[$reponame]}"
    test -n "$mirror_url" || DIE "MIRROR_URLS[$reponame] is empty! Check file $conf."

    test -n "$folder" || folder=.
    test -d "$folder" || DIE "folder $folder does not exist!"
    test -r "$folder/endeavouros.db" || {
        echo2 "$(basename $0): nothing to see here."
        return
    }

    mirror_url="$(echo "$mirror_url" | sed "s|\$arch|$arch|")"
    mirror_url="$(echo "$mirror_url" | sed "s|\$repo|$repo|")"

    echo2 "$reponame:"
    remotefilelist="$(curl -s "$mirror_url" | grep -Pw 'tar|db|files' | sed 's|.*<a href="\([^"]*\)".*|\1|')"
    test -n "$remotefilelist" || DIE "cannot fetch package info from $mirror_url."

    pushd "$folder" >/dev/null

    packages="$(ls -1 *.pkg.tar.*)"
    repofiles="$(echo endeavouros.{db,files}{,.tar.xz})"

    local handled=0
    local count=0
    local -r count_pkgs=$(echo "$packages" | wc -l)
    local -r count_repofiles=$(printf "%s\n" $repofiles | wc -l)
    local -r count_state=1
    local  count_filelist=0

    [ "$has_filelist" = "yes" ] && count_filelist=1
    count=$((count_pkgs + count_repofiles + count_filelist + count_state))

    # Check files. 'NameCheck' is very fast but unreliable. 'ContentsCheck' is quite slow but reliable.
    local tmpdir=$(mktemp -d)
    ContentsCheck $repofiles                 # This should make sure all is OK ! (?)
    ContentsCheck state
    [ "$has_filelist" = "yes" ] && ContentsCheck repofiles.txt

    case "$mode" in
        fast | fastall)
            NameCheck $packages
            ;;
        slow)
            ContentsCheck $packages
            ;;
        optimized)
            for arg in $packages ; do
                case "$arg" in
                    # icon theme packages are very big
                    paper-icon-theme-*.pkg.tar.xz | paper-icon-theme-*.pkg.tar.zst | \
                    arc-x-icons-theme-*.pkg.tar.xz | arc-x-icons-theme-*.pkg.tar.zst)
                        NameCheck "$arg" ;;
                    *)
                        ContentsCheck "$arg" ;;
                esac
            done
            ;;
    esac
    rm -rf $tmpdir
    [ $only_fails = yes ] && echo2 ""

    if [ -z "$problemfiles" ] ; then
        echo2i "==> No issues."
    else
        echo2i "==> Not yet ready."

        #echo2i "==> Problematic files: "
        #for arg in $problemfiles ; do
        #    echoi -l3 "$arg"
        #done
    fi
    popd >/dev/null
}

Usage() {
    cat <<EOF >&2

$progname: Check the validity of EndeavourOS packages in mirrors.

Usage: $progname [parameters]
parameters:
    --slow             Check contents of each file.
    --fast             Check just the name of all package files (default).
    --optimized        Check contents of some files, and names of other files.
    --only-fails, -f   Show only failed file checks.
    --verbose, -v      Show checks in detail. See also: --only-fails.
    --reponame=X       Use given name as the mirror name (default: all known mirrors).
    --no-filelist      Don't generate list of latest files into the repo.
    --show-params      Show all available parameters as a list (for completion support).
    <folder-name>      Name of the local folder that contains package files.
                       If not given, current folder is used.

EOF
}

Main()
{
    # use assets.conf
    # local assets_conf=./assets.conf
    # test -r $assets_conf || DIE "cannot find local file $assets_conf."

    local -r progname=${0##*/}
    local has_filelist=no # "$(grep "^local USE_GENERATED_FILELIST=" $assets_conf | sed 's|^.*="\([yesno]*\)".*$|\1|')"
    local mode=fast   # show, optimized, fast
    local folder=""
    local reponame=""
    local only_fails=yes
    local arg
    local arch=""
    local -r DEFAULT_MIRROR_NAME=alpix.eu
    local -r list=/etc/pacman.d/endeavouros-mirrorlist
    local hide_fallback=no
    local supported_params=(
        "<folder-name>"
        --slow
        --fast
        --optimized
        --only-fails -f
        --verbose -v
        --reponame=
        --no-filelist
        --show-params
    )
    for arg in "$@" ; do
        case "$arg" in
            --show-params)       echo "${supported_params[*]}" ; exit ;;
            --show-mirror-nicks) hide_fallback=yes ;;
        esac
    done

    local MIRROR_NAMES=(
        # Get names from the endeavouros-mirrorlist with ad hoc algorithm.
        # Assume endeavouros-mirrorlist includes the up-to-date mirrors in comments (lines of "#Server = https://...").
        # This implementation picks only lines starting with "#Server = ".
        $(grep "^#Server = " $list | awk '{print $NF}')
    )
    if [ ${#MIRROR_NAMES[0]} -eq 0 ] ; then
        [ $hide_fallback = no ] && echo2 "==> $progname: no full mirrorlist -> fallback to only configured mirrors"
        MIRROR_NAMES=(
            # This implementation picks only lines starting with "Server = ".
            $(grep "^Server = " $list | awk '{print $NF}')
        )
    fi
    [ ${#MIRROR_NAMES[0]} -gt 0 ] || DIE "MIRROR_NAMES[0] is empty! Check file $conf."

    MIRROR_NAMES=(
        $(
            printf "%s\n" "${MIRROR_NAMES[@]}" |
                sed -E -e 's|^https://([^/]+)/.*|\1|' -e 's|^.*mirror\.||' -e 's|^.*mirrors\.||' -e 's|^www\.||' -e 's|^ftp\.||'
        )
    )

    for arg in "$@" ; do
        case "$arg" in
            --reponame=*) reponame="${arg#*=}" ;;
            --slow) mode=slow ;;
            --fast) mode=fast ;;
            --optimized) mode=optimized ;;
            --only-fails | -f) only_fails=yes ;;
            --verbose | -v)    only_fails=no ;;
            --no-filelist) has_filelist=no ;;   # no repofiles.txt
            --show-mirror-nicks) echo "${MIRROR_NAMES[*]}" ; exit  ;; # for bash completion
            -*) DIE "unsupported parameter '$arg'" ;;
            *) folder="$arg" ;;
        esac
    done

    [ "$(printf "%s\n" "${MIRROR_NAMES[@]}" | grep "$DEFAULT_MIRROR_NAME" )" ] || DIE "$DEFAULT_MIRROR_NAME was not found in $list"

    [ -n "$folder" ] && cd "$folder"  # NOTE: special cd !!

    if [ -r assets.conf ] && [ -d PKG_ARCHIVE ] ; then
        arch=x86_64                # we are in the original build folder
    else
        arch=${PWD##*/}    # x86_64 or aarch64
        case "$arch" in
            x86_64 | aarch64) ;;
            *) DIE "must run this program at an architecture specific folder (e.g. repo/endeavouros/x86_64)." ;;
        esac
        case "$(grep "/endeavouros-team/" ../../.git/config)" in
            *https://github.com/endeavouros-team/repo | *https://github.com/endeavouros-team/repo.git) ;;
            *) DIE "this folder is not in endeavouros-team repo" ;;
        esac
    fi

    # Internet URL to the mirror's folder where EndeavourOS repo packages are.
    # Note: some mirrors require a trailing slash '/' character.
    #
    declare -A MIRROR_URLS

    local name xx

    for name in "${MIRROR_NAMES[@]}" ; do
        xx="$(grep "^Server = " $list | grep -iw "$name" | head -n1 | awk '{print $3}')/"  # note: / at end !!
        if [ "$xx" = "/" ] ; then
            xx="$(grep "^#Server = " $list | grep -iw "$name" | head -n1 | awk '{print $3}')/"  # note: / at end !!
        fi
        if [ "$xx" != "/" ] ; then
            MIRROR_URLS[$name]="$xx"
        else
            echo "==> /etc/eos-mirrorcheck.conf: error: mirror name '$name' refers to no URL $list!" >&2
        fi
    done

    case "$reponame" in
        "")
            reponame="$DEFAULT_MIRROR_NAME"
            OneMirror
            ;;
        all)
            #mode=fastall
            for reponame in "${MIRROR_NAMES[@]}" ; do
                OneMirror
            done
            ;;
        *)
            if [ -n "$(grep "^Server = " $list | grep -iw "$reponame")" ] ; then
                OneMirror
            else
                cat <<EOF
Sorry, '$reponame' is unknown, or the URL is not active in file $list.
EOF
            fi
            ;;
    esac
}

Main "$@"