build-ubunturepo

#!/bin/bash
# Update GSA Ubuntu Repositories or create a local repository
#
# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com)
# Revisor: Arif Ali (aali@ocf.co.uk)
#
# 
# Getting Started:
#  - Check out the xcat-core svn repository (either the trunk or a branch) into
#    a dir called <rel>/src/xcat-core, where <rel> is the same as the release dir it will be
#    uploaded to in sourceforge (e.g. devel, or 2.3).
#  - You probably also want to put root's pub key from the build machine onto sourceforge for
#    the upload user listed below, so you don't have to keep entering pw's.  You can do this
#    at https://sourceforge.net/account/ssh
#  - make sure reprepro is installed on the build machine
#  - Run this script from the local svn repository you just created.  It will create the other directories that are needed.

# Usage:  attr=value attr=value ... ./build-ubunturepo { -c | -d }
#               PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release.  This does not
#                                       actually build xcat, just uploads the most recent snap build to https://sourceforge.net/projects/xcat/files/xcat/ .
#                                       If not specified, a snap build is assumed, which uploads to https://sourceforge.net/projects/xcat/files/yum/
#                                       or https://sourceforge.net/projects/xcat/files/aix/.
#               PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this build is
#                                       a GA candidate build, not to be released yet.  This will result in the tarball being uploaded to
#                                       https://sourceforge.net/projects/xcat/files/yum/ or https://sourceforge.net/projects/xcat/files/aix/
#                                       (but the tarball file name will be like a released tarball, not a snap build).  When you are ready to
#                                       release this build, use PROMOTE=1 without PREGA
#               BUILDALL=1 - build all rpms, whether they changed or not.  Should be used for snap builds that are in prep for a release.
#               UP=0 or UP=1 - override the default upload behavior
#               LOG=<filename> - provide an LOG file option to redirect some output into log file
#
# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to 
#                                "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA
#                             2. run ./build-ubunturepo -d
#
#                             3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" 
#                                related to the path of this script
############################
printusage()
{
    printf "Usage: %s {-c | -d} \n" $(basename $0) >&2
    echo "    -c : Build the xcat core packages and create the repo"
    echo "    -d : Create the xcat dep repo. Building the xcat dep packages can refer \"build-debs-all\" from svn"
}
# For the purpose of getting the distribution name
. /etc/lsb-release

# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name
for i in $*; do
        echo $i | grep '='
        if [ $? != 0 ];then
            continue
        fi
        # upper case the variable name
        varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2`
        export $varstring
done

# Supported distributions
dists="saucy trusty utopic"

c_flag=            # xcat-core (trunk-delvel) path
d_flag=            # xcat-dep (trunk) path
r_flag=            #genesis base rpm package path

while getopts 'cdr:' OPTION
do
    case $OPTION in
	c)    c_flag=1
	        ;;
	d)    d_flag=1
	        ;;
        r)    r_flag=1
                genesis_rpm_path="$OPTARG"
                ;;
	?)  printusage 
	    exit 2
	    ;;
    esac
done
shift $(($OPTIND - 1))

if [ -z "$c_flag" -a -z "$d_flag" ];then
    printusage
    exit 2
fi

if [ "$c_flag" -a "$d_flag" ];then
    printusage
    exit 2
fi



uploader="litingt"
# Find where this script is located to set some build variables
old_pwd=`pwd`
cd `dirname $0`
curdir=`pwd`

#define the dep source code path, core build target path and dep build target path
local_core_repo_path="$curdir/../../xcat-core"
local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep"

#define the url used for creating the source list file
#define the upload dir used for uploading packages
sf_repo_url="https://sourceforge.net/projects/xcat/files/ubuntu"
sf_dir="/home/frs/project/x/xc/xcat"

#use flock to only one person build at the same time
# Get a lock, so can not do 2 builds at once
exec 8>/var/lock/xcatbld.lock
if ! flock -n 8; then
    echo "Can't get lock /var/lock/xcatbld.lock.  Someone else must be doing a build right now.  Exiting...."
    exit 1
fi

export HOME=/root

WGET_CMD="wget"
if [ ! -z ${LOG} ]; then 
    WGET_CMD="wget -o ${LOG}"
fi

#sync the gpg key to the build machine local
gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux
mkdir -p $HOME/.gnupg
for key_name in pubring.gpg secring.gpg trustdb.gpg; do
    if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then
        rm -f $HOME/.gnupg/$key_name
        ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name
        chmod 600 $HOME/.gnupg/$key_name
    fi
done

if [ "$c_flag" ]
then
    # strip the /src/xcat-core from the end of the dir to get the next dir up and use as the release
    if [ -z "$REL" ]; then
        t=${curdir%/src/xcat-core}
        REL=`basename $t`
    fi

    ver=`cat Version`
    if [ "$PROMOTE" != 1 ]; then
        code_change=0
        update_log=''
        #get the version
        if [ "$REL" = "xcat-core" ];then
            git_flag=1
            REL=`git rev-parse --abbrev-ref HEAD`
            if [ "$REL" = "master" ]; then
                REL="devel"
            fi
            if [ -z "$GITUP" ];then
                update_log=../coregitup
                echo "git pull > $update_log"
                git pull > $update_log
            else
                update_log=$GITUP
            fi

            if ! grep -q  'Already up-to-date' $update_log; then
                code_change=1
            fi
        else
            git_flag=0
            if [ -z "$SVNUP" ];	then
                update_log=../coresvnup
                echo "svn up > $update_log"
                svn up > $update_log
            else
                update_log=$SVNUP
            fi

            if ! grep -q 'At revision' $update_log;then
                code_change=1
            fi
        fi
        ver=`cat Version`
        short_ver=`cat Version|cut -d. -f 1,2`
        short_short_ver=`cat Version|cut -d. -f 1`

        package_dir_name=debs$REL
        #TODO: define the core path and tarball name
        tarball_name="core-debs-snap.tar.bz2"

        if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then
            echo "Nothing new detected"
            exit 0
        fi

        echo "###############################"
        echo "# Building xcat-core packages #"
        echo "###############################"

        #the package type:  local | snap | alpha
        #the build introduce stirng
        pkg_type="snap"
        build_string="Snap_Build"
        cur_date=`date +%Y%m%d%H%M`
        pkg_version="${short_ver}-${pkg_type}${cur_date}"
    
        if [ ! -d ../../$package_dir_name ];then
            mkdir -p "../../$package_dir_name"
        fi
        packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent"
        target_archs=(amd64 ppc64el)
        for file in `echo $packages`
        do
            file_low=`echo $file | tr '[A-Z]' '[a-z]'`
            if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" ]; then
                target_archs="amd64 ppc64el"
            else
                target_archs="all"
            fi
            for target_arch in `echo $target_archs`
            do
                if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then
                    rm -f ../../$package_dir_name/${file_low}_*.$target_arch.deb
                    #genesis scripts package, don't remove genesis amd64 files
                    #rm -f ../../$package_dir_name/${file_low}-amd64_*.deb
                    cd $file
                    dch -v $pkg_version -b -c debian/changelog $build_string
                    if [ "$target_arch" = "all" ]; then
                        dpkg-buildpackage -uc -us
                    else
                        dpkg-buildpackage -uc -us -a$target_arch
                    fi
                    rc=$?
                    if [ $rc -gt 0 ]; then
                        echo "Error: $file build package failed exit code $rc"
                        exit $rc
                    fi
                    cd -
                    find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf
                    find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf
                    mv ${file_low}* ../../$package_dir_name/
                fi
            done
        done

        find ../../$package_dir_name/* ! -name *.deb | xargs rm -f
    else
        if [ "$REL" = "xcat-core" ];then
            git_flag=1
            REL=`git rev-parse --abbrev-ref HEAD`
            if [ "$REL" = "master" ]; then
                REL="devel"
            fi
        fi
        package_dir_name=debs$REL
    fi

    if [ "$PROMOTE" = 1 ]; then
        upload_dir="xcat-core"
        tar_name="xcat-core-$ver.tar.bz2"
    else
        upload_dir="core-snap"
        tar_name="core-debs-snap.tar.bz2"
    fi

    echo "#################################"
    echo "# Creating xcat-core repository #"
    echo "#################################"

    #clean the repo directory
    if [ -e $local_core_repo_path ]; then
    	rm -rf $local_core_repo_path
    fi
    mkdir -p $local_core_repo_path
    cd $local_core_repo_path
    mkdir conf

    for dist in $dists; do
       if [ "$dist" = "trusty" ] || [ "$dist" = "utopic" ]; then
           tmp_out_arch="amd64 ppc64el"
       else
           tmp_out_arch="amd64"
       fi
       cat << __EOF__ >> conf/distributions
Origin: xCAT internal repository
Label: xcat-core bazaar repository
Codename: $dist
Architectures: $tmp_out_arch
Components: main
Description: Repository automatically genereted conf 
SignWith: yes

__EOF__
     done

    cat << __EOF__ > conf/options
verbose
ask-passphrase
basedir .
__EOF__

    #import the deb packages into the repo
    amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"`
    all_files=`ls ../$package_dir_name/*.deb`
    for dist in $dists; do
      if [ "$dist" = "trusty" ] || [ "$dist" = "utopic" ]; then
          deb_files=$all_files
      else
          deb_files=$amd_files
      fi
      for file in $deb_files; do
	reprepro -b ./ includedeb $dist $file;
      done
    done
    #create the mklocalrepo script
    cat << '__EOF__' > mklocalrepo.sh
. /etc/lsb-release
cd `dirname $0`
host_arch=`uname -m`
if [ "$host_arch" != "ppc64le" ];then
    host_arch="amd64"
else
    host_arch="ppc64el"
fi
echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list
__EOF__

    chmod 775 mklocalrepo.sh

    #create the xcat-core.list file 
    #echo "deb ${sf_repo_url}/${REL}/${upload_dir}/ precise main" > xcat-core.list

    cd ../
    if ! grep xcat /etc/group ; then
        groupadd xcat
    fi     

    chgrp -R root xcat-core
    chmod -R g+w xcat-core

    #build the tar ball
    tar -hjcf $tar_name xcat-core
    chgrp root $tar_name
    chmod g+w $tar_name

    if [ ! -e core-snap ]; then
        ln -s xcat-core core-snap
    fi

    #
    # remove the changes to changelog since the files get changed and created in xcat-core..
    # left over files not commited to git will prevent switching from branch to branch ...
    echo "Cleaning up modified changelog files..."
    cleanup_oldpwd=`pwd`
    # go to the xcat-core git repo directory 
    #
    cd $local_core_repo_path
    cd ../xcatbuild/xcat-core
    find ./ -name "changelog" -print | xargs git checkout
    cd $cleanup_oldpwd

    # Decide whether to upload or not
    if [ -n "$UP" ] && [ "$UP" == 0 ]; then
        echo "No need to upload"
        cd $old_pwd
        exit 0
    fi

    #upload the deb packages
    if [ "$REL" = "devel" -o "$PREGA" != 1 ]; then
        i=0
        echo "Uploading RPMs from $upload_dir to ${sf_dir}/ubuntu/${REL}/ ..."
        while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete $upload_dir ${uploader},xcat@web.sourceforge.net:${sf_dir}/ubuntu/${REL}/
        do : ; done
    fi

    #upload the tar ball
    #for the GA build, upload to https://sourceforge.net/projects/xcat/files/xcat/<version>.x_Ubuntu/
    #for other scenario, upload to https://sourceforge.net/projects/xcat/files/ubuntu/<version>
    if [ "$PROMOTE" = 1 -a "$REL" != "devel" -a "$PREGA" != 1 ]; then
        i=0
        echo "Uploading $tar_name to ${sf_dir}/xcat/${REL}.x_Ubuntu/ ..."
        while [ $((i+=1)) -le 5 ] && ! rsync -v $tar_name ${uploader},xcat@web.sourceforge.net:${sf_dir}/xcat/${REL}.x_Ubuntu/
        do : ; done
    else
        i=0
        echo "Uploading $tar_name to ${sf_dir}/ubuntu/${REL}/ ..."
        while [ $((i+=1)) -le 5 ] && ! rsync -v $tar_name ${uploader},xcat@web.sourceforge.net:${sf_dir}/ubuntu/${REL}/
        do : ; done
    fi
    
    cd $old_pwd   
fi

if [ "$d_flag" ]
then
    echo "################################"
    echo "# Creating xcat-dep repository #"
    echo "################################"


    #the path of ubuntu xcat-dep deb packages on GSA
    GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep"
 

    if [ ! -d $GSA ]; then
       echo "build-ubunturepo:  It appears that you do not have gsa installed to access the xcat-dep pkgs."
       exit 1;
    fi

    # Sync from the GSA master copy of the dep rpms
    echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..."
    rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../


    #clean all old files
    if [ -e $local_dep_repo_path ];then
        rm -rf $local_dep_repo_path
    fi
    mkdir -p $local_dep_repo_path
    cd $local_dep_repo_path
    mkdir conf


    #create the conf/distributions file
    for dist in $dists; do
       if [ "$dist" = "trusty" ] || [ "$dist" = "utopic" ]; then
           tmp_out_arch="amd64 ppc64el"
       else 
           tmp_out_arch="amd64"
       fi
        cat << __EOF__ >> conf/distributions
Origin: xCAT internal repository
Label: xcat-dep bazaar repository
Codename: $dist
Architectures: $tmp_out_arch
Components: main
Description: Repository automatically genereted conf 
SignWith: yes

__EOF__
    done

    cat << __EOF__ > conf/options
verbose
ask-passphrase
basedir .
__EOF__

    #import the deb packages into the repo
    amd_files=`ls ../debs/*.deb | grep -v "ppc64el"`
    all_files=`ls ../debs/*.deb`
    for dist in $dists; do
      if [ "$dist" = "trusty" ] || [ "$dist" = "utopic" ]; then
          deb_files=$all_files
      else
          deb_files=$amd_files
      fi
      for file in $deb_files; do
	reprepro -b ./ includedeb $dist $file;
      done
    done

    cat << '__EOF__' > mklocalrepo.sh
. /etc/lsb-release
cd `dirname $0`
host_arch=`uname -m`
if [ "$host_arch" != "ppc64le" ];then
    host_arch="amd64"
else
    host_arch="ppc64el"
fi
echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list
__EOF__

    chmod 775 mklocalrepo.sh

    #echo ""deb ${sf_repo_url}/xcat-dep/ precise main"" > xcat-dep.list

    cd ..
    if ! grep xcat /etc/group ; then
        groupadd xcat
    fi

    chgrp -R root xcat-dep
    chmod -R g+w xcat-dep

    #create the tar ball
    dep_tar_name=xcat-dep-ubuntu-snap`date +%Y%m%d`.tar.bz
    tar -hjcf $dep_tar_name xcat-dep
    chgrp root $dep_tar_name
    chmod g+w $dep_tar_name

    # remove the changes to changelog since the builds are occuring in xcat-core, will prevent
    # switching from branch to branch without a new clone of the repository
    find ./ -name "changelog" -print | xargs git checkout

    if [ -n "$UP" ] && [ "$UP" == 0 ];then
        echo "No need to upload the dep packages"
        cd $old_pwd
        exit 0
    fi

    #upload the dep packages
    i=0
    echo "Uploading debs from xcat-dep to ${sf_dir}/ubuntu/xcat-dep/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep ${uploader},xcat@web.sourceforge.net:${sf_dir}/ubuntu/
         do : ; done
    #upload the tarball
    i=0
    echo "Uploading $dep_tar_name to ${sf_dir}/xcat-dep/2.x_Ubuntu/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -v $dep_tar_name ${uploader},xcat@web.sourceforge.net:${sf_dir}/xcat-dep/2.x_Ubuntu/
    do : ; done

    #upload the README file
    cd debs
    i=0
    echo "Uploading README to ${sf_dir}/xcat-dep/2.x_Ubuntu/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -v README ${uploader},xcat@web.sourceforge.net:${sf_dir}/xcat-dep/2.x_Ubuntu/
    do : ; done

    cd $old_pwd
fi
exit 0