rsync-netbsd-cvs

#! /bin/sh
#
#	rsync-netbsd-cvs - rsync the CVS repo from NetBSD
#
# also updates local working directories....
#

PATH=$PATH:/usr/pkg/sbin:/usr/pkg/bin:/usr/local/sbin:/usr/local/bin
export PATH

LC_ALL=C
export LC_ALL

#
#	local configuration
#

# location of the local repository
#
# WARNING: this pathname needs a trailing slash!
#
REPO_DIR="/vcs/master/m-NetBSD/main/"

# A list of files possibly containing locally added directories which
# need to be re-created in the REPO_DIR after rsync will have removed
# them.  (since they don't exist in the remote master repository)
#
# n.b.:  any whitespace or special characters in pathnames must be quoted or
# escaped in a manner compatible with xargs(1).
#
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/pkgsrc/LOCALDIRS"		# not normally used
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/pkgsrc-current/LOCALDIRS"
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/src/LOCALDIRS"			# not normally used
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/src-current/LOCALDIRS"
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/xsrc/LOCALDIRS"		# not normally used
LOCALDIRS_LISTS="${LOCALDIRS_LISTS} /usr/xsrc-current/LOCALDIRS"

# NOTE:  If you probably want the following files in ${REPO_DIR}:
#
# main/CVSROOT/config:
#	LockDir=/tmp
#	LogHistory=
#	LocalKeyword=NetBSD=CVSHeader
#	KeywordExpand=iNetBSD,Name,Date,Author,Revision
#	KeywordExpand=eId,CVSHeader,Header
#
# and you'll want to make them immutable:  'chflags schg'

# places to run "cvs update" after the rsync is done
#
# N.B.:  You probably do not want to rely on the automatic "checkout" feature
# below, at least not for large modules like NetBSD, pkgsrc, etc., as the
# resulting email will be WAY too large.
#
WORK_DIRS=""
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/src-1.6"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/src-4"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/xsrc-4"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/src-5"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/src-7"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/src-8"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/xsrc-5"
WORK_DIRS="${WORK_DIRS} /work/NetBSD/othersrc"
WORK_DIRS="${WORK_DIRS} /work/NetBSD/src"
WORK_DIRS="${WORK_DIRS} /work/NetBSD/xsrc"
WORK_DIRS="${WORK_DIRS} /work/NetBSD/pkgsrc"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/htdocs"
#WORK_DIRS="${WORK_DIRS} /home/wwwadmin/htdocs-netbsd.robohack.ca"
# XXX sadly these are NOT in the anoncvs collection
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/CVSROOT"
#WORK_DIRS="${WORK_DIRS} /work/NetBSD/localsrc"

# places to run "cvs -n update" after the rsync is done
#
WORK_DIRS_CHECK=""
#WORK_DIRS_CHECK="${WORK_DIRS_CHECK} /work/woods/m-NetBSD-pkgsrc-2006Q4"
#WORK_DIRS_CHECK="${WORK_DIRS_CHECK} /work/woods/m-NetBSD-pkgsrc-2007Q1"
#WORK_DIRS_CHECK="${WORK_DIRS_CHECK} /work/woods/m-NetBSD-pkgsrc-2010Q1"

#
#	remote configuration
#

# master NetBSD site....
#
SITE="anoncvs.netbsd.org"
METHOD="rsync://"
DIRECTORY="/cvsroot/"		# aka the "module", with a trailing slash!
EXTERNAL=""
#
# The "main" sub-directory doesn't exist at anoncvs.netbsd.org, so we
# must include it in the local repo parameter to rsync
#
LOCAL_REPO=${REPO_DIR}

# old site (was another name for the same IP as ftp.netbsd.org):
#
#    "Rsync service for the "anoncvs" module has moved from ftp.netbsd.org to
#    anoncvs.netbsd.org.
#
#    Please see http://mail-index.netbsd.org/netbsd-users/2004/03/29/0010.html
#    for important details."
#
#SITE="rsync.netbsd.org"
#METHOD="rsync://"
#DIRECTORY="/anoncvs/"
#EXTERNAL=""
#
# The "main" sub-directory _does_ exist at rsync.netbsd.org...
#
#LOCAL_REPO=${REPO_DIR%%main/}

# alternate site
#
#SITE="rsync.jp.netbsd.org"
#METHOD="rsync://"
#DIRECTORY="/anoncvs/"
#
# Does the "main" sub-directory exist at rsync.jp.netbsd.org?
#
#LOCAL_REPO=${REPO_DIR}
#EXTERNAL=""

# alternate site showing use of ssh
#
#SITE=""
#METHOD="woods@whome.planix.com:"
#DIRECTORY="/u5/NetBSD-CVS/"
#LOCAL_REPO=${REPO_DIR}
#EXTERNAL="-e ssh --blocking-io"

#
#	END all configuration!
#

just_make_localdirs=false
just_rsync=false

# set it to "echo" for debugging
DEBUG_ECHO=""
#
# use ${DQ} in front of and after any quoted multi-token parameters on
# command lines that are preceded by "${DEBUG_ECHO} \"
#
DQ=""
RSYNC_DEBUG=""
CVS_DEBUG=""

EX_OK=0
EX_USAGE=2

# XXX we should have "just rsync" and "just update" options too!
#
OPT_STRING=dhlr

argv0=$(basename $0)

mktable ()
{
	sed				\
		-e '/^[ 	]*#/d'	\
		-e 's/#.*$//'		\
		-e '/^[ 	]*$/d'	\
	${1+"$@"}
}

# re-create directories in the REPO_DIR that we have local (but of
# course uncommitted) new files within...
#
# xxx hmmm....:   | tr '\n' '\0' | xargs -0 install -d -m 775 --
#
make_localdirs ()
{
	for local_dirs_list in ${LOCALDIRS_LISTS}; do
		if [ -s ${local_dirs_list} ] ; then
			mktable ${local_dirs_list} | sed -e "s|^|${REPO_DIR}|"
		fi
	done | sort -u | xargs \
	$DEBUG_ECHO \
	install -d -m 775 --

	unset local_dirs_list
}

usage_error ()
{
	echo "Usage: $argv0 [-${OPT_STRING}]" 1>&2
	exit $EX_USAGE
}

usage_help ()
{
	echo "Usage: $argv0 [-${OPT_STRING}]"
	echo "	-d	enable debug output for testing"
	echo "	-h	print this help message"
	echo "	-l	just run the make_localdirs part"
	echo "	-r	just run rsync and make_localdirs (don't update work directories)"
	exit $EX_OK
}

while getopts $OPT_STRING OPT; do
	case "$OPT" in
	d)
		# for testing
		#
		RSYNC_DEBUG="-vv -n"
		CVS_DEBUG="-n"
		DEBUG_ECHO=echo
		DQ="'"
		;;
	h)
		usage_help
		;;
	l)
		just_make_localdirs=true
		;;
	r)
		just_rsync=true
		;;
	\?)
		usage_error
		;;
	esac
done
shift $(($OPTIND - 1))
[ $# -ne 0 ] && usage_error

if $just_make_localdirs; then
	make_localdirs
	exit 0
fi

# WARNING:  this script could fail to interlock properly if it can be
# invoked simultaneously with more than one name!  Don't do that!
# 
LOCKDIR=/tmp/$(basename $0).lock

if [ -e $LOCKDIR ] ; then
	echo "It appears there's already an instance of $0 running..." 1>&2
	exit 1
fi

if mkdir $LOCKDIR ; then
	: #got it!
else
	echo "Oops, just missed grabbing $LOCKDIR!" 1>&2
	exit 1
fi

cp /var/log/rsync-netbsd-cvs /var/log/rsync-netbsd-cvs.old

(
	while true; do
		printf "START:%s: rsync\n" "$(date '+%Y/%m/%d-%T')"
		rsync ${RSYNC_DEBUG} -rlptHS --timeout=86000 -vz ${EXTERNAL} --stats --delete --delete-after --exclude '#cvs.lock' ${METHOD}${SITE}${DIRECTORY} ${LOCAL_REPO}
		rc=$?
		#

		# Note that E#23 happens normally when rsync can't
		# delete the files I've made purposefully immutable
		# specifically so that it can't delete them.
		#
		# Also presumably if the remote server had a problem
		# or there was some other filesystem error on this end
		# then we should just give up and report what happened
		# and wait for tomorrow to try again
		#
		if [ $rc -eq 0 -o $rc -eq 23 ]; then
			break
		else
			echo ""
			printf "FAILED:%s: rsync exited with code: %d\n" "$(date '+%Y/%m/%d-%T')" $rc
		fi
		# no newline in case someone's doing a 'tail -f'....
		printf "PAUSE:%s: " "$(date '+%Y/%m/%d-%T')"
		sleep 60
		echo "RESTART"
	done
	printf "DONE:%s: rsync\n" "$(date '+%Y/%m/%d-%T')"
) > /var/log/rsync-netbsd-cvs 2>&1

# this produces the Cron report from the raw log by ignoring any line
# which ends in a '/' (presumably it's just one of the directories
# that was updated)
#
grep -v '/$' /var/log/rsync-netbsd-cvs
echo ""

make_localdirs

if $just_rsync; then
	rmdir $LOCKDIR
	exit 0
fi

for work_dir in $WORK_DIRS; do
	echo "================================================================"
	printf "\nSTART:%s: cvs update $work_dir\n\n" "$(date '+%Y/%m/%d-%T')"
	if [ ! -d $work_dir ]; then
		case ${work_dir} in
		/work/NetBSD*)
			work_dir_repo="-d /cvs/master/m-NetBSD/main"
			work_dir_dir=${work_dir%/*}
			work_dir_base=${work_dir##*/}
			work_dir_module=${work_dir_base%-*}
			# xxx this probably won't work very well
			case ${work_dir_module} in
			pkgsrc)
				case ${work_dir_base} in
				*-*)
					work_dir_tag="-r ${work_dir_base}"
					;;
				esac
				;;
			xsrc|src)
				case ${work_dir_base} in
				*-*)
					work_dir_tag="-r netbsd-${work_dir_base##*-}"
					;;
				esac
				;;
			esac
			;;
		esac
		echo "WARNING: there is no ${work_dir} yet!"
		cd ${work_dir_dir} && cvs ${work_dir_repo} ${CVS_DEBUG} -u -q checkout -P ${work_dir_tag} -d ${work_dir_base} ${work_dir_module}
	else
		cd $work_dir && cvs ${CVS_DEBUG} -u -q update -P -d
	fi
	printf "\nDONE:%s: updating $work_dir\n\n" "$(date '+%Y/%m/%d-%T')"
	if [ -x /usr/pkg/bin/lintpkgsrc -a -f $work_dir/packages-to-upgrade -a $(basename $work_dir) = "pkgsrc" ] ; then
		/usr/pkg/bin/lintpkgsrc -P $work_dir -i | tee $work_dir/packages-to-upgrade.new
		if [ -s $work_dir/packages-to-upgrade.new ] ; then
			mv -f $work_dir/packages-to-upgrade.new  $work_dir/packages-to-upgrade
		fi
	fi
done

for work_dir in $WORK_DIRS_CHECK; do
	printf "\nSTART:%s: cvs -n update $work_dir\n\n" "$(date '+%Y/%m/%d-%T')"
	if [ ! -d $work_dir ]; then
		echo "WARNING: there is no ${work_dir} yet!"
	else
		cd $work_dir && cvs -u -nq update -P -d 2>&1 | fgrep -v 'cvs update: New directory'
	fi
	printf "\nDONE:%s: checking $work_dir\n\n" "$(date '+%Y/%m/%d-%T')"
done

if [ ! -d $LOCKDIR ] ; then
	echo "OOPS!!!!  $LOCKDIR went missing!!!!" 1>&2
	exit 1
fi
rmdir $LOCKDIR

exit $?