sync_repo-v1.3
by
kanarip
—
last modified
Jan 30, 2008 03:46 AM
sync_repo
—
text/x-sh,
15Kb
File contents
#!/bin/bash
# Copyright 2006 Jeroen 'kanarip' van Meeuwen <kanarip@kanarip.com>
# Licensed under the GPL
# Version 1.2
# This script is to synchronize Fedora and third party repositories
# Features:
# - RSYNC
###
### User Configuration
###
# Check for a newer version of this script (we'll just be verbose)
check_script_version="yes"
# Limit the bandwidth (kbytes/second) (Make a parameter)
#bwlimit=70
# Base directory (No trailing slash)
# basedir="/data/os"
basedir="/data/os"
#
# When syncing release/arch, sync all stable (core, updates, extras, livna, core-source, updates-source, extras-source)
# Parameter passed to the script will be similar to: fc_5(_i386)
#
# Get the available repositories from cat /etc/yum.repos.d/* | grep -E "^\["
declare -a repos
repos[0]="core"
repos[1]="core-debuginfo"
repos[2]="core-source"
repos[3]="development"
repos[4]="development-debuginfo"
repos[5]="development-source"
repos[6]="extras-development"
repos[7]="extras-development-debuginfo"
repos[8]="extras-development-source"
repos[9]="extras"
repos[10]="extras-debuginfo"
repos[11]="extras-source"
repos[12]="legacy-updates"
repos[13]="legacy-testing"
repos[14]="updates"
repos[15]="updates-debuginfo"
repos[16]="updates-source"
repos[17]="updates-testing"
repos[18]="updates-testing-debuginfo"
repos[19]="updates-testing-source"
repos[20]="livna"
repos[21]="livna-testing"
repos[22]="livna-debuginfo"
repos[23]="livna-development"
repos[24]="livna-development-debuginfo"
repos[25]="livna-development-source"
repos[26]="livna-source"
repos[27]="livna-testing-debug"
repos[28]="livna-testing-source"
#repos[29]="atrpms-stable"
#repos[30]="atrpms-bleeding"
#repos[31]="atrpms-testing"
#repos[32]="freshrpms"
#repos[33]="freshrpms-testing"
#repos[34]="dribble"
#repos[35]="dribble-devel"
#repos[36]="dribble-source"
declare -a repo_local
repo_local[0]="$basedir/fedora/core/\${release}/\${arch}/"
repo_local[1]="$basedir/fedora/core/\${release}/\${arch}/debug/"
repo_local[2]="$basedir/fedora/core/\${release}/source/"
repo_local[3]="$basedir/fedora/core/development/\${arch}/"
repo_local[4]="$basedir/fedora/core/development/\${arch}/debug/"
repo_local[5]="$basedir/fedora/core/development/SRPMS/"
repo_local[6]="$basedir/fedora/extras/development/\${arch}/"
repo_local[7]="$basedir/fedora/extras/development/\${arch}/debug/"
repo_local[8]="$basedir/fedora/extras/development/SRPMS/"
repo_local[9]="$basedir/fedora/extras/\${release}/\${arch}/"
repo_local[10]="$basedir/fedora/extras/\${release}/\${arch}/debug/"
repo_local[11]="$basedir/fedora/extras/\${release}/SRPMS/"
repo_local[12]="$basedir/fedora/\${release}/updates/\${arch}/"
repo_local[13]="$basedir/fedora/\${release}/updates-testing/\${arch}/"
repo_local[14]="$basedir/fedora/core/updates/\${release}/\${arch}/"
repo_local[15]="$basedir/fedora/core/updates/\${release}/\${arch}/debug/"
repo_local[16]="$basedir/fedora/core/updates/\${release}/SRPMS/"
repo_local[17]="$basedir/fedora/core/updates/testing/\${release}/\${arch}/"
repo_local[18]="$basedir/fedora/core/updates/testing/\${release}/\${arch}/debug/"
repo_local[19]="$basedir/fedora/core/updates/testing/\${release}/SRPMS/"
repo_local[20]="$basedir/livna/\${release}/\${arch}/"
repo_local[21]="$basedir/livna/testing/\${release}/\${arch}/"
repo_local[22]="$basedir/livna/\${release}/\${arch}/debug/"
repo_local[23]="$basedir/livna/development/\${arch}/"
repo_local[24]="$basedir/livna/development/\${arch}/debug/"
repo_local[25]="$basedir/livna/development/SRPMS/"
repo_local[26]="$basedir/livna/\${release}/SRPMS/"
repo_local[27]="$basedir/livna/testing/\${release}/\${arch}/debug/"
repo_local[28]="$basedir/livna/testing/\${release}/SRPMS/"
#repo_local[29]="$basedir/atrpms/stable/\${release}/\${arch}/"
#repo_local[30]="$basedir/atrpms/bleeding/\${release}/\${arch}/"
#repo_local[31]="$basedir/atrpms/testing/\${release}/\${arch}/"
#repo_local[32]="$basedir/freshrpms/\${release}/"
#repo_local[33]="$basedir/freshrpms/testing/\${release}/"
#repo_local[34]="$basedir/dribble/\${release}/\${arch}/"
#repo_local[35]="$basedir/dribble/devel/\${release}/\${arch}/"
#repo_local[36]="$basedir/dribble/\${release}/SRPMS/"
declare -a releases
releases[0]="3"
releases[1]="4"
releases[2]="5"
releases[3]="6"
releases[4]="7"
declare -a architectures
archs[0]="i386"
archs[1]="x86_64"
archs[2]="ppc"
#archs[3]="ia64"
#archs[4]="ppc64"
#archs[5]="s390"
#archs[6]="s390x"
declare -a synchronize
declare -a sync_repos
declare -a sync_releases
declare -a sync_archs
usage() {
echo "Usage:"
echo " $0 [OPTIONS] repository"
}
rm "$basedir"/sync_repo.last > /dev/null 2>&1
random_from_list() {
# When passed multiple parameters, choose one (random)
if [ $# -eq 0 ]; then
echo "Invalid number of parameters to random_from_list(). Continuing."
return
fi
declare -a parameters
parameter_count=0
while [ -n "$1" ]; do
parameter_count=$[ $parameter_count + 1 ]
parameters[$parameter_count]="$1"
shift
done
echo "${parameters[$[ $RANDOM % $parameter_count + 1 ]]}"
}
random_mirror_select() {
# Given 1 parameter
# Mirrorlist Filename to get the random mirror from
echo $(random_from_list $(cat $1))
}
requireutil() {
# Pass 1 parameter to test if the command exists, or the RPM package
# is installed
#
# Example:
# requireutil coreutils
# requireutil lftp
if [ ! $# -eq 1 ]; then
# Invalid parameters
echo "Invalid number of parameters to requireutil(). Aborting."
exit 1
elif [ -z $(which $1) ]; then
# No command $1 found
echo "Required command $1 not found. Aborting."
exit 1
elif [ -z `rpm -qv $1 | grep -v "not installed"` ]; then
# No RPM package $1 installed
echo "Required RPM package $1 not installed. Aborting."
exit 1
fi
}
find_local_dir_for_repo() {
# Find the local repo directory!!
w_repo_local=0
while [ $w_repo_local -lt ${#repos[@]} ]; do
if [ "$1" == "${repos[$w_repo_local]}" ]; then
repo=$1
target=$(eval echo ${repo_local[$w_repo_local]})
break
fi
w_repo_local=$[ $w_repo_local + 1 ]
done
echo $(eval echo $target)
}
repo_synchronize() {
# Given 3 or 4 parameters
# $1 - Repository to mirror
# $2 - Release
# $3 - Arch
# $4 - Repository base url (Local)
#
# Example:
# repo_synchonize core 6 i386 /data/os/fedora/\${repo}/\${release}/\${arch}/
repo=$1
release=$2
arch=$3
if [ $# -eq 4 ]; then
target=$(eval echo $4)
elif [ $# -lt 3 ]; then
echo "Invalid number of parameters to repo_synchonize() -> $# ($1 and $2)"
exit 1
else
target=$(find_local_dir_for_repo $1)
fi
requireutil rsync
if [ ! -d $target ]; then
echo "No such directory: $target. Creating it."
mkdir -p $target
fi
if [ -n $bwlimit ]; then
bwlimit_option="--bwlimit=$bwlimit"
else
bwlimit_option=""
fi
status="starting"
pass=0
if [ -f "mirrorlist_$1" ]; then
while [ "$status" != "passed" -a $pass -lt 3 ]; do
mirror=$(eval echo $(random_mirror_select mirrorlist_${1}))
case "$mirror" in
http://*)
;;
ftp://*)
;;
rsync://*)
echo Repository ${repo}
echo From $mirror
echo To $target
# From man rsync
# rsync -a == rsync -rlptgoD
# We don't need to 'go'
# -r recursive
# -l copy symlinks (don't descend)
# -p preserve permissions
# -t preserve times
# -D preserve devices and specials
# -H copy hardlinks (don't descend)
# -v verbose
# -z compress file data during transfer
env RSYNC_PROXY=proxy.kanarip.com:3128 \
rsync -aDvz \
--progress \
--link-dest=/data/os/fedora/ \
${bwlimit_option} \
${rsync_delete_option} \
${rsync_exclude_option} \
--exclude="debug/" \
--exclude="testing/" \
$mirror $target
# ALSO, CONSIDER
#
# --link-dest=DIR
# This option behaves like --copy-dest, but unchanged files are
# hard linked from DIR to the destination directory. The files
# must be identical in all preserved attributes (e.g. permissions,
# possibly ownership) in order for the files to be linked
# together. An example:
#
# rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/
#
# Beginning in version 2.6.4, multiple --link-dest directories may
# be provided, which will cause rsync to search the list in the
# order specified for an exact match. If a match is found that
# differs only in attributes, a local copy is made and the
# attributes updated. If a match is not found, a basis file from
# one of the DIRs will be selected to try to speed up the trans-
# fer.
#
# If DIR is a relative path, it is relative to the destination
# directory. See also --compare-dest and --copy-dest.
#
# Note that rsync versions prior to 2.6.1 had a bug that could
# prevent --link-dest from working properly for a non-super-user
# when -o was specified (or implied by -a). You can work-around
# this bug by avoiding the -o option when sending to an old rsync.
#
if [ $? -eq 0 ]; then
status="passed"
fi
;;
*)
echo "Unrecognized type of mirror: $mirror"
;;
esac
pass=$[ $pass + 1 ]
done
else
echo "No mirrorlist file mirrorlist_$1"
fi
}
build_command() {
if [ ! -z "$3" ]; then
w_releases=0
while [ $w_releases -lt ${#releases[@]} ]; do
if [ "$2" == "${releases[$w_releases]}" ]; then
sync_releases[${#sync_releases[@]}]=$2
break
fi
w_releases=$[ $w_releases + 1 ]
done
w_archs=0
while [ $w_archs -lt ${#archs[@]} ]; do
if [ "$3" == "${archs[$w_archs]}" ]; then
sync_archs[${#sync_archs[@]}]=$3
break
fi
w_archs=$[ $w_archs + 1 ]
done
elif [ ! -z "$2" ]; then
# $2 is valuable, $3 is not set, so sync all archs
w_releases=0
while [ $w_releases -lt ${#releases[@]} ]; do
if [ "$2" == "${releases[$w_releases]}" ]; then
sync_releases[${#sync_releases[@]}]=$2
break
fi
w_releases=$[ $w_releases + 1 ]
done
sync_archs=( $(echo ${archs[@]:0}) )
else
# $2 isn't there, $3 is not set, so sync all releases and archs
sync_releases=( $(echo ${releases[@]:0}) )
sync_archs=( $(echo ${archs[@]:0}) )
fi
}
## From here we parse arguments and run sync commands and such
declare -a match_repos_to
if [ $# -gt 0 ]; then
while [ ! -z "$1" ]; do
case "$1" in
--devel)
sync_devel="yes"
;;
--source)
sync_source="yes"
;;
--testing)
sync_testing="yes"
;;
--debuginfo)
sync_debuginfo="yes"
;;
--legacy)
sync_legacy="yes"
;;
--nocore)
sync_core="no"
;;
--noupdates)
sync_updates="no"
;;
--noextras)
sync_extras="no"
;;
--nolivna)
sync_livna="no"
;;
--noatrpms)
sync_atrpms="no"
;;
--nodribble)
sync_dribble="no"
;;
--nofreshrpms)
sync_freshrpms="no"
;;
--norpmforge)
sync_rpmforge="no"
;;
--bwlimit)
if [ -n $2 ]; then
bwlimit=$2
shift
else
usage
exit
fi
;;
--excludenoyum)
rsync_exclude_option="--exclude=os/images/ --exclude=os/isolinux/ --exclude=os/stylesheet-images/ --exclude=iso/"
;;
--delete)
rsync_delete_option="--delete"
;;
--delete-after)
rsync_delete_option="--delete-after"
;;
fc-*|fc*)
input=$(echo $1 | tr "-" " ")
build_command $input
;;
--all|all)
build_command fc
;;
*)
usage
;;
esac
shift
done
#else
# repo_synchronize extras 6 i386 /data/os/fedora/\${repo}/\${release}/\${arch}/
fi
sync_repos=( $(echo ${repos[@]:0}) )
if [ -z "${sync_devel}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*development*}) )
fi
if [ -z "${sync_source}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*source*}) )
fi
if [ -z "${sync_testing}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*testing*}) )
fi
if [ -z "${sync_debuginfo}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*debuginfo*}) )
fi
if [ -z "${sync_legacy}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*legacy*}) )
fi
if [ ! -z "${sync_core}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*core*}) )
fi
if [ ! -z "${sync_updates}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*updates*}) )
fi
if [ ! -z "${sync_extras}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*extras*}) )
fi
if [ ! -z "${sync_livna}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*livna*}) )
fi
if [ ! -z "${sync_atrpms}" ]; then
sync_repos=( $(echo ${sync_repos[@]##*atrpms*}) )
fi
w_repos=0
while [ $w_repos -lt ${#sync_repos[@]} ]; do
w_releases=0
while [ $w_releases -lt ${#sync_releases[@]} ]; do
w_archs=0
while [ $w_archs -lt ${#sync_archs[@]} ]; do
case "${sync_repos[$w_repos]}" in
*source)
sync_num=0
no_add=0
while [ $sync_num -lt ${#synchronize[@]} ]; do
if [ "repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} source" == "${synchronize[$sync_num]}" ]; then
no_add=1
fi
sync_num=$[ $sync_num + 1 ]
done
if [ $no_add -eq 0 ]; then
synchronize[${#synchronize[@]}]="repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} source"
# echo synchronize[${#synchronize[@]}]="repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} source"
fi
;;
*)
sync_num=0
no_add=0
while [ $sync_num -lt ${#synchronize[@]} ]; do
if [ "repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} ${sync_archs[$w_archs]}" == "${synchronize[$sync_num]}" ]; then
no_add=1
fi
sync_num=$[ $sync_num + 1 ]
done
if [ $no_add -eq 0 ]; then
synchronize[${#synchronize[@]}]="repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} ${sync_archs[$w_archs]}"
# echo synchronize[${#synchronize[@]}]="repo_synchronize ${sync_repos[$w_repos]} ${sync_releases[$w_releases]} ${sync_archs[$w_archs]}"
fi
;;
esac
w_archs=$[ $w_archs + 1 ]
done
w_releases=$[ $w_releases + 1 ]
done
w_repos=$[ $w_repos + 1 ]
done
#exit
i=0
while [ $i -lt ${#synchronize[@]} ]; do
eval ${synchronize[$i]}
i=$[ $i + 1 ]
done
echo
echo Calculating disk usage for each module
echo
w_repos=0
while [ $w_repos -lt ${#repos[@]} ]; do
repo=${repos[$w_repos]}
w_releases=0
while [ $w_releases -lt ${#releases[@]} ]; do
release=${releases[$w_releases]}
case "${repos[$w_repos]}" in
*source)
arch="source"
if [ -d $(eval echo ${repo_local[$w_repos]}) ]; then
diskspace[$w_repos]=$(du -s $(eval echo ${repo_local[$w_repos]}) --exclude=debug/ --exclude=testing/)
if [ ! -z "${diskspace[$w_repos]}" ]; then
if [ $(echo ${diskspace[$w_repos]} | cut -d' ' -f1) -lt 20 ]; then
rm -rf $(echo ${diskspace[$w_repos]} | cut -d' ' -f2)
else
echo ${repos[$w_repos]} ${diskspace[$w_repos]}
fi
fi
else
diskspace[$w_repos]=0
fi
;;
*)
w_archs=0
while [ $w_archs -lt ${#archs[@]} ]; do
arch=${archs[$w_archs]}
if [ -d $(eval echo ${repo_local[$w_repos]}) ]; then
diskspace[$w_repos]=$(du -s $(eval echo ${repo_local[$w_repos]}) --exclude=debug/ --exclude=testing/)
if [ ! -z "${diskspace[$w_repos]}" ]; then
if [ $(echo ${diskspace[$w_repos]} | cut -d' ' -f1) -lt 20 ]; then
rm -rf $(echo ${diskspace[$w_repos]} | cut -d' ' -f2)
else
echo ${repos[$w_repos]} ${diskspace[$w_repos]}
fi
fi
else
diskspace[$w_repos]=0
fi
w_archs=$[ $w_archs + 1 ]
done
;;
esac
w_releases=$[ $w_releases + 1 ]
done
w_repos=$[ $w_repos + 1 ]
done
Forgot your password?
New user?
