sync_repo-v1.2

File contents

#!/bin/bash

# Copyright 2006 Jeroen 'kanarip' van Meeuwen <kanarip@kanarip.com>
# Licensed under the GPL
# Version 1.2

# This script is to synchronize Fedora and third party repositories
# Features:
# - RSYNC

###
### User Configuration
###

# Check for a newer version of this script (we'll just be verbose)
check_script_version="yes"

# Limit the bandwidth (kbytes/second) (Make a parameter)
#bwlimit=70

# Base directory (No trailing slash)
# basedir="/data/os"
basedir="/data/os"

#
# When syncing release/arch, sync all stable (core, updates, extras, livna, core-source, updates-source, extras-source)
# Parameter passed to the script will be similar to: fc_5(_i386)
#

# Get the available repositories from cat /etc/yum.repos.d/* | grep -E "^\["
declare -a repos
repos[0]="core"
repos[1]="core-debuginfo"
repos[2]="core-source"
repos[3]="development"
repos[4]="development-debuginfo"
repos[5]="development-source"
repos[6]="extras-development"
repos[7]="extras-development-debuginfo"
repos[8]="extras-development-source"
repos[9]="extras"
repos[10]="extras-debuginfo"
repos[11]="extras-source"
repos[12]="legacy-updates"
repos[13]="legacy-testing"
repos[14]="updates"
repos[15]="updates-debuginfo"
repos[16]="updates-source"
repos[17]="updates-testing"
repos[18]="updates-testing-debuginfo"
repos[19]="updates-testing-source"
repos[20]="livna"
repos[21]="livna-testing"
repos[22]="livna-debuginfo"
repos[23]="livna-development"
repos[24]="livna-development-debuginfo"
repos[25]="livna-development-source"
repos[26]="livna-source"
repos[27]="livna-testing-debug"
repos[28]="livna-testing-source"

declare -a repo_local
repo_local[0]="$basedir/fedora/core/\${release}/\${arch}/"
repo_local[1]="$basedir/fedora/core/\${release}/\${arch}/debug/"
repo_local[2]="$basedir/fedora/core/\${release}/source/SRPMS/"
repo_local[3]="$basedir/fedora/core/development/\${arch}/"
repo_local[4]="$basedir/fedora/core/development/\${arch}/debug/"
repo_local[5]="$basedir/fedora/core/development/SRPMS/"
repo_local[6]="$basedir/fedora/extras/development/\${arch}/"
repo_local[7]="$basedir/fedora/extras/development/\${arch}/debug/"
repo_local[8]="$basedir/fedora/extras/development/SRPMS/"
repo_local[9]="$basedir/fedora/extras/\${release}/\${arch}/"
repo_local[10]="$basedir/fedora/extras/\${release}/\${arch}/debug/"
repo_local[11]="$basedir/fedora/extras/\${release}/SRPMS/"
repo_local[12]="$basedir/fedora/\${release}/updates/\${arch}/"
repo_local[13]="$basedir/fedora/\${release}/updates-testing/\${arch}/"
repo_local[14]="$basedir/fedora/core/updates/\${release}/\${arch}/"
repo_local[15]="$basedir/fedora/core/updates/\${release}/\${arch}/debug/"
repo_local[16]="$basedir/fedora/core/updates/\${release}/SRPMS/"
repo_local[17]="$basedir/fedora/core/updates/testing/\${release}/\${arch}/"
repo_local[18]="$basedir/fedora/core/updates/testing/\${release}/\${arch}/debug/"
repo_local[19]="$basedir/fedora/core/updates/testing/\${release}/SRPMS/"
repo_local[20]="$basedir/livna/\${release}/\${arch}/"
repo_local[21]="$basedir/livna/testing/\${release}/\${arch}/"
repo_local[22]="$basedir/livna/\${release}/\${arch}/debug/"
repo_local[23]="$basedir/livna/development/\${arch}/"
repo_local[24]="$basedir/livna/development/\${arch}/debug/"
repo_local[25]="$basedir/livna/development/SRPMS/"
repo_local[26]="$basedir/livna/\${release}/SRPMS/"
repo_local[27]="$basedir/livna/testing/\${release}/\${arch}/debug/"
repo_local[28]="$basedir/livna/testing/\${release}/SRPMS/"

declare -a releases
releases[0]="3"
releases[1]="4"
releases[2]="5"
releases[3]="6"
releases[4]="7"

declare -a architectures
archs[0]="i386"
archs[1]="x86_64"
archs[2]="ppc"
archs[3]="source"
archs[4]="ia64"
archs[5]="ppc64"
archs[6]="s390"
archs[7]="s390x"

declare -a synchronize
declare -a sync_repos
declare -a sync_releases
declare -a sync_archs

usage() {
echo "Usage:"
echo "    $0 [OPTIONS] repository"

}

rm "$basedir"/sync_repo.last > /dev/null 2>&1

random_from_list() {
	# When passed multiple parameters, choose one (random)
	if [ $# -eq 0 ]; then
		echo "Invalid number of parameters to random_from_list(). Continuing."
		return
	fi
	
	declare -a parameters
	
	parameter_count=0
	
	while [ -n "$1" ]; do
		parameter_count=$[ $parameter_count + 1 ]
		parameters[$parameter_count]="$1"
		shift
	done
	
	echo "${parameters[$[ $RANDOM % $parameter_count + 1 ]]}"
}

random_mirror_select() {
	# Given 1 parameter
	# Mirrorlist Filename to get the random mirror from
	echo $(random_from_list $(cat $1))
}

requireutil() {

	# Pass 1 parameter to test if the command exists, or the RPM package
	# is installed
	#
	# Example:
	# requireutil coreutils
	# requireutil lftp
	
	if [ ! $# -eq 1 ]; then
		# Invalid parameters
		echo "Invalid number of parameters to requireutil(). Aborting."
		exit 1
	elif [ -z $(which $1) ]; then
		# No command $1 found
		echo "Required command $1 not found. Aborting."
		exit 1
	elif [ -z `rpm -qv $1 | grep -v "not installed"` ]; then
		# No RPM package $1 installed
		echo "Required RPM package $1 not installed. Aborting."
		exit 1
	fi
}

find_local_dir_for_repo() {

	# Find the local repo directory!!
	w_repo_local=0
	while [ $w_repo_local -lt ${#repos[@]} ]; do
		if [ "$1" == "${repos[$w_repo_local]}" ]; then
			repo=$1
			target=$(eval echo ${repo_local[$w_repo_local]})
			break
		fi
		w_repo_local=$[ $w_repo_local + 1 ]
	done

	echo $(eval echo $target)
}

repo_synchronize() {

	# Given 3 or 4 parameters
	# $1 - Repository to mirror
	# $2 - Release
	# $3 - Arch
	# $4 - Repository base url (Local)
	#
	# Example:
	# repo_synchonize core 6 i386 /data/os/fedora/\${repo}/\${release}/\${arch}/
	
	repo=$1
	release=$2
	arch=$3

	if [ $# -eq 4 ]; then
		target=$(eval echo $4)
	elif [ $# -lt 3 ]; then
		echo "Invalid number of parameters to repo_synchonize() -> $# ($1 and $2)"
		exit 1
	else
		target=$(find_local_dir_for_repo $1)
	fi
	
	requireutil rsync

	if [ ! -d $target ]; then
		echo "No such directory: $target. Creating it."
		mkdir -p $target
	fi

	if [ -n $bwlimit ]; then
		bwlimit_option="--bwlimit=$bwlimit"
	else
		bwlimit_option=""
	fi
	
	status="starting"
	pass=0
	
	if [ -f "mirrorlist_$1" ]; then
		while [ "$status" != "passed" -a $pass -lt 3 ]; do
			mirror=$(eval echo $(random_mirror_select mirrorlist_${1}))

			case "$mirror" in
				http://*)
				;;
				ftp://*)
				;;
				rsync://*)
					echo Repository ${repo} 
					echo From $mirror
					echo To $target

					rsync -rlptDHqz ${bwlimit_option} ${rsync_delete_option} ${rsync_exclude_option} --exclude="debug/" --exclude="testing/" $mirror $target

					if [ $? -eq 0 ]; then
						status="passed"
					fi
				;;
				*)
					echo "Unrecognized type of mirror: $mirror"
				;;
			esac

			pass=$[ $pass + 1 ]
		done
	else
		echo "No mirrorlist file"
	fi
		
}

build_command() {
	if [ ! -z "$3" ]; then
		w_releases=0
		while [ $w_releases -lt ${#releases[@]} ]; do
			if [ "$2" == "${releases[$w_releases]}" ]; then
				sync_releases[${#sync_releases[@]}]=$2
				break
			fi
			w_releases=$[ $w_releases + 1 ]
		done
		
		w_archs=0
		while [ $w_archs -lt ${#archs[@]} ]; do
			if [ "$3" == "${archs[$w_archs]}" ]; then
				sync_archs[${#sync_archs[@]}]=$3
				break
			fi
			w_archs=$[ $w_archs + 1 ]
		done
	elif [ ! -z "$2" ]; then
		# $2 is valuable, $3 is not set, so sync all archs
		
		w_releases=0
		while [ $w_releases -lt ${#releases[@]} ]; do
			if [ "$2" == "${releases[$w_releases]}" ]; then
				sync_releases[${#sync_releases[@]}]=$2
				break
			fi
			w_releases=$[ $w_releases + 1 ]
		done
    sync_archs=( $(echo ${archs[@]:0}) )
	else
		# $2 isn't there, $3 is not set, so sync all releases and archs
		sync_releases=( $(echo ${releases[@]:0}) )
    sync_archs=( $(echo ${archs[@]:0}) )
	fi
}

## From here we parse arguments and run sync commands and such

declare -a match_repos_to
if [ $# -gt 0 ]; then
  while [ ! -z "$1" ]; do
	  case "$1" in
	    --devel)
	    	sync_devel="yes"
	    ;;
	    --source)
	    	sync_source="yes"
	    ;;
	    --testing)
	    	sync_testing="yes"
	    ;;
	    --debuginfo)
	    	sync_debuginfo="yes"
	    ;;
	    --legacy)
	    	sync_legacy="yes"
	    ;;
	    --nocore)
	    	sync_core="no"
	    ;;
	    --noupdates)
	    	sync_updates="no"
	    ;;
	    --noextras)
	    	sync_extras="no"
	    ;;
	    --nolivna)
	    	sync_livna="no"
	    ;;
	    --excludenoyum)
        rsync_exclude_option="--exclude=os/images/ --exclude=os/isolinux/ --exclude=os/stylesheet-images/ --exclude=iso/"
      ;;
      --delete)
      	rsync_delete_option="--delete"
      ;;
	    fc_*|fc*)
	      input=$(echo $1 | tr "-" " ")
	      build_command $input
	    ;;
	    --all|all)
	    	build_command fc
	    ;;
	    *)
	      usage
			;;
	  esac
	 	shift
	done
#else
#  repo_synchronize extras 6 i386 /data/os/fedora/\${repo}/\${release}/\${arch}/
fi

sync_repos=( $(echo ${repos[@]:0}) )

if [ -z "${sync_devel}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*development*}) )
fi
if [ -z "${sync_source}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*source*}) )
fi
if [ -z "${sync_testing}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*testing*}) )
fi
if [ -z "${sync_debuginfo}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*debuginfo*}) )
fi
if [ -z "${sync_legacy}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*legacy*}) )
fi
if [ ! -z "${sync_core}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*core*}) )
fi
if [ ! -z "${sync_updates}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*updates*}) )
fi
if [ ! -z "${sync_extras}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*extras*}) )
fi
if [ ! -z "${sync_livna}" ]; then
	sync_repos=( $(echo ${sync_repos[@]##*livna*}) )
fi

w_repos=0
while [ $w_repos -lt ${#sync_repos[@]} ]; do
	w_releases=0
	while [ $w_releases -lt ${#sync_releases[@]} ]; do
		w_archs=0
		while [ $w_archs -lt ${#sync_archs[@]} ]; do
			synchronize[${#synchronize[@]}]="repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} ${sync_archs[$w_archs]}"
			w_archs=$[ $w_archs + 1 ]
		done
		w_releases=$[ $w_releases + 1 ]
	done
	w_repos=$[ $w_repos + 1 ]
done

i=0
while [ $i -lt ${#synchronize[@]} ]; do
	eval ${synchronize[$i]}
	i=$[ $i + 1 ]
done

echo Calculating disk usage for each module
w_repos=0
while [ $w_repos -lt ${#repos[@]} ]; do
  repo=${repos[$w_repos]}
	w_releases=0
	while [ $w_releases -lt ${#releases[@]} ]; do
		release=${releases[$w_releases]}
    case "${repos[$w_repos]}" in
			*source)
				arch="source"
				if [ -d $(eval echo ${repo_local[$w_repos]}) ]; then
					diskspace[$w_repos]=$(du -s $(eval echo ${repo_local[$w_repos]}) --exclude=debug/ --exclude=testing/)
					echo ${repos[$w_repos]} ${diskspace[$w_repos]}
				else
					diskspace[$w_repos]=0
				fi
			;;
			*)
				w_archs=0
				while [ $w_archs -lt ${#archs[@]} ]; do
					arch=${archs[$w_archs]}
					if [ -d $(eval echo ${repo_local[$w_repos]}) ]; then
						diskspace[$w_repos]=$(du -s $(eval echo ${repo_local[$w_repos]}) --exclude=debug/ --exclude=testing/)
						echo ${repos[$w_repos]} ${diskspace[$w_repos]}
					else
						diskspace[$w_repos]=0
					fi
					w_archs=$[ $w_archs + 1 ]
				done
			;;
		esac
		w_releases=$[ $w_releases + 1 ]
	done
	w_repos=$[ $w_repos + 1 ]
done