#!/bin/bash
# main autocluster script
#
# Copyright (C) Andrew Tridgell 2008
# Copyright (C) Martin Schwenke 2008
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see .
##BEGIN-INSTALLDIR-MAGIC##
# There are better ways of doing this but not if you still want to be
# able to run straight out of a git tree. :-)
if [ -f "$0" ]; then
installdir="`dirname \"$0\"`"
else
autocluster=`which $0`
installdir="`dirname \"$autocluster\"`"
fi
##END-INSTALLDIR-MAGIC##
####################
# show program usage
usage ()
{
cat <
options:
-c specify config file (default is "config")
EOF
releases=$(list_releases)
usage_smart_display \
defconf "WITH_RELEASE" "" \
"" "specify preset options for a release using a version string. Possible values are: ${releases}."
cat < execute and exit (advanced debugging)
-x enable script debugging
--dump dump config settings and exit
configuration options:
EOF
usage_config_options
cat <&2
exit 1
}
###############################
# Indirectly call a function named by ${1}_${2}
call_func () {
local func="$1" ; shift
local type="$1" ; shift
local f="${func}_${type}"
if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null ; then
"$f" "$@"
else
f="${func}_DEFAULT"
if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null ; then
"$f" "$type" "$@"
else
die "No function defined for \"${func}\" \"${type}\""
fi
fi
}
# Note that this will work if you pass "call_func f" because the first
# element of the node tuple is the node type. Nice... :-)
for_each_node ()
{
local n
for n in $NODES ; do
"$@" $(IFS=: ; echo $n)
done
}
hack_one_node_with ()
{
local filter="$1" ; shift
local node_type="$1"
local ip_offset="$2"
local name="$3"
local ctdb_node="$4"
$filter
local item="${node_type}:${ip_offset}${name:+:}${name}${ctdb_node:+:}${ctdb_node}"
nodes="${nodes}${nodes:+ }${item}"
}
# This also gets used for non-filtering iteration.
hack_all_nodes_with ()
{
local filter="$1"
local nodes=""
for_each_node hack_one_node_with "$filter"
NODES="$nodes"
}
##############################
# common node creation stuff
create_node_COMMON ()
{
local NAME="$1"
local ip_offset="$2"
local template_file="${3:-$NODE_TEMPLATE}"
IPNUM=$(($FIRSTIP + $ip_offset))
DISK="${VIRTBASE}/${CLUSTER}/${NAME}.qcow2"
mkdir -p $VIRTBASE/$CLUSTER tmp
echo "Creating the disk"
rm -f "$DISK"
qemu-img create -b "$VIRTBASE/$BASENAME.img" -f qcow2 "$DISK"
mount_disk $DISK
setup_base
setup_network
unmount_disk
set_macaddrs $CLUSTER $ip_offset
UUID=`uuidgen`
echo "Creating $NAME.xml"
substitute_vars $template_file tmp/$NAME.xml
# install the XML file
$VIRSH undefine $NAME > /dev/null 2>&1 || true
$VIRSH define tmp/$NAME.xml
}
# Provides an easy way of removing nodes from $NODE.
create_node_null () {
:
}
###############################
##############################
hack_nodes_functions=
register_nodes_hack ()
{
local hack="$1"
hack_nodes_functions="${hack_nodes_functions}${hack_nodes_functions:+ }${hack}"
}
expand_nodes () {
# Expand out any abbreviations in NODES.
local ns=""
local n
for n in $NODES ; do
local t="${n%:*}"
local ips="${n#*:}"
case "$ips" in
*,*)
local i
for i in ${ips//,/ } ; do
ns="${ns}${ns:+ }${t}:${i}"
done
;;
*-*)
local i
for i in $(seq ${ips/-/ }) ; do
ns="${ns}${ns:+ }${t}:${i}"
done
;;
*)
ns="${ns}${ns:+ }${n}"
esac
done
NODES="$ns"
# Apply nodes hacks. Some of this is about backward compatibility
# but the hacks also fill in the node names and whether they're
# part of the CTDB cluster. The order is the order that
# configuration modules register their hacks.
for n in $hack_nodes_functions ; do
$n
done
if [ -n "$NUMNODES" ] ; then
# Attempt to respect NUMNODES. Reduce the number of CTDB
# nodes to NUMNODES.
local numnodes=$NUMNODES
hack_filter ()
{
if [ "$ctdb_node" = 1 ] ; then
if [ $numnodes -gt 0 ] ; then
numnodes=$(($numnodes - 1))
else
node_type="null"
ctdb_node=0
fi
fi
}
hack_all_nodes_with hack_filter
[ $numnodes -gt 0 ] && \
die "Can't not use NUMNODES to increase the number of nodes over that specified by NODES. You need to set NODES instead - please read the documentation."
fi
# Check IP addresses for duplicates.
local ip_offsets=":"
# This function doesn't modify anything...
get_ip_offset ()
{
[ "${ip_offsets/${ip_offset}}" != "$ip_offsets" ] && \
die "Duplicate IP offset in NODES - ${node_type}:${ip_offset}"
ip_offsets="${ip_offsets}${ip_offset}:"
}
hack_all_nodes_with get_ip_offset
}
##############################
create_cluster_hooks=
register_create_cluster_hook ()
{
local hook="$1"
create_cluster_hooks="${create_cluster_hooks}${create_cluster_hooks:+ }${hook}"
}
create_cluster() {
CLUSTER="$1"
[ -n "${CLUSTER//[A-Za-z0-9]}" ] && \
die "Cluster names should be restricted to the characters A-Za-z0-9. \
Some cluster filesystems have problems with other characters."
mkdir -p tmp
# Rework the NODES list
expand_nodes
# Build /etc/hosts and hack the names of the ctdb nodes
hosts_line_hack_name ()
{
# Ignore nodes without names (e.g. "null")
[ "$node_type" != "null" -a -n "$name" ] || return 0
local sname=""
if [ "$ctdb_node" = 1 ] ; then
num_ctdb_nodes=$(($num_ctdb_nodes + 1))
sname="${CLUSTER}n${num_ctdb_nodes}"
echo "$IPBASE.0.$(($FIRSTIP + $ip_offset)) ${name}.${ld} ${sname}.${ld} ${sname}"
name="$sname"
else
echo "$IPBASE.0.$(($FIRSTIP + $ip_offset)) ${name}.${ld} ${name}"
fi
}
hosts_file="tmp/hosts.$CLUSTER"
{
local num_ctdb_nodes=0
local ld=$(echo $DOMAIN | tr A-Z a-z)
echo "# autocluster $CLUSTER"
hack_all_nodes_with hosts_line_hack_name
echo
} >$hosts_file
# Build /etc/ctdb/nodes
ctdb_nodes_line ()
{
[ "$ctdb_node" = 1 ] || return 0
echo "$IPBASE.0.$(($FIRSTIP + $ip_offset))"
}
nodes_file="tmp/nodes.$CLUSTER"
hack_all_nodes_with ctdb_nodes_line >$nodes_file
mkdir -p $VIRTBASE/$CLUSTER $KVMLOG
# Run hooks before creating nodes.
local n
for n in $create_cluster_hooks ; do
$n
done
# Create the actual nodes
for_each_node call_func create_node
echo "Cluster $CLUSTER created"
echo "You may want to add this to your /etc/hosts file:"
cat $hosts_file
}
###############################
# test the proxy setup
test_proxy() {
export http_proxy=$WEBPROXY
wget -O /dev/null $INSTALL_SERVER || \
die "Your WEBPROXY setting \"$WEBPROXY\" is not working"
echo "Proxy OK"
}
###################
# create base image
create_base() {
NAME="$BASENAME"
DISK="$VIRTBASE/$NAME.img"
mkdir -p $KVMLOG
echo "Testing WEBPROXY $WEBPROXY"
test_proxy
echo "Creating the disk"
qemu-img create -f $BASE_FORMAT "$DISK" $DISKSIZE
rm -rf tmp
mkdir -p mnt tmp tmp/ISO
setup_timezone
echo "Creating kickstart file from template"
substitute_vars "$KICKSTART" "tmp/ks.cfg"
if [ $INSTALLKEY = "--skip" ]; then
cat < /dev/null || true
nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
killall -9 -q nbd-client || true
nbd-client localhost 1300 $NBD_DEVICE > /dev/null 2>&1 || true &
sleep 1
}
# disconnect nbd
disconnect_nbd() {
echo "Disconnecting nbd"
sync; sync
nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
killall -9 -q nbd-client || true
killall -q $QEMU_NBD || true
}
# mount a qemu image via nbd
mount_disk() {
connect_nbd $1
echo "Mounting disk $1"
mount_ok=0
for i in `seq 1 5`; do
mount -o offset=32256 $NBD_DEVICE mnt && {
mount_ok=1
break
}
umount mnt 2>/dev/null || true
sleep 1
done
[ $mount_ok = 1 ] || die "Failed to mount $1"
[ -d mnt/root ] || {
echo "Mounted directory does not look like a root filesystem"
ls -latr mnt
exit 1
}
}
# unmount a qemu image
unmount_disk() {
echo "Unmounting disk"
sync; sync;
umount mnt || umount mnt || true
disconnect_nbd
}
# setup the files from $BASE_TEMPLATES/, substituting any variables
# based on the config
setup_base() {
umask 022
echo "Copy base files"
for f in `cd $BASE_TEMPLATES && find . \! -name '*~'`; do
if [ -d "$BASE_TEMPLATES/$f" ]; then
mkdir -p mnt/"$f"
else
substitute_vars "$BASE_TEMPLATES/$f" "mnt/$f"
fi
chmod --reference="$BASE_TEMPLATES/$f" "mnt/$f"
done
# this is needed as git doesn't store file permissions other
# than execute
chmod 600 mnt/etc/ssh/*key mnt/root/.ssh/*
chmod 700 mnt/etc/ssh mnt/root/.ssh mnt/root
if [ -r "$HOME/.ssh/id_rsa.pub" ]; then
echo "Adding $HOME/.ssh/id_rsa.pub to ssh authorized_keys"
cat "$HOME/.ssh/id_rsa.pub" >> mnt/root/.ssh/authorized_keys
fi
if [ -r "$HOME/.ssh/id_dsa.pub" ]; then
echo "Adding $HOME/.ssh/id_dsa.pub to ssh authorized_keys"
cat "$HOME/.ssh/id_dsa.pub" >> mnt/root/.ssh/authorized_keys
fi
echo "Adjusting grub.conf"
local o="$EXTRA_KERNEL_OPTIONS" # For readability.
sed -e "s/console=ttyS0,19200/console=ttyS0,115200/" \
-e "s/ nodmraid//" -e "s/ nompath//" \
-e "s/quiet/divider=10${o:+ }${o}/g" mnt/boot/grub/grub.conf -i.org
}
# setup various networking components
setup_network() {
echo "Setting up networks"
cat $hosts_file >>mnt/etc/hosts
echo "Setting up /etc/ctdb/nodes"
mkdir -p mnt/etc/ctdb
cp $nodes_file mnt/etc/ctdb/nodes
[ "$WEBPROXY" = "" ] || {
echo "export http_proxy=$WEBPROXY" >> mnt/etc/bashrc
}
if [ -n "$NFSSHARE" -a -n "$NFS_MOUNTPOINT" ] ; then
echo "Enabling nfs mount of $NFSSHARE"
mkdir -p "mnt$NFS_MOUNTPOINT"
echo "$NFSSHARE $NFS_MOUNTPOINT nfs intr" >> mnt/etc/fstab
fi
mkdir -p mnt/etc/yum.repos.d
substitute_vars "$YUM_TEMPLATE" mnt/etc/yum.repos.d/$(basename $YUM_TEMPLATE)
}
setup_timezone() {
[ -z "$TIMEZONE" ] && {
[ -r /etc/timezone ] && {
TIMEZONE=`cat /etc/timezone`
}
[ -r /etc/sysconfig/clock ] && {
. /etc/sysconfig/clock
TIMEZONE="$ZONE"
}
TIMEZONE="${TIMEZONE// /_}"
}
[ -n "$TIMEZONE" ] || \
die "Unable to determine TIMEZONE - please set in config"
}
# substite a set of variables of the form @@XX@@ for the shell
# variables $XX in a file.
#
# Indirect variables @@@XX@@@ (3 ats) specify that the variable should
# contain a filename whose contents are substituted, with variable
# substitution applied to those contents. If filename starts with '|'
# it is a command instead - however, quoting is extremely fragile.
substitute_vars() {(
infile="${1:-/dev/null}" # if empty then default to /dev/null
outfile="$2" # optional
instring=$(cat $infile)
# Handle any indirects by looping until nothing changes.
# However, only handle 10 levels of recursion.
count=0
while : ; do
outstring=$(_substitute_vars "$instring" "@@@")
[ $? -eq 0 ] || die "Failed to expand template $infile"
[ "$instring" = "$outstring" ] && break
count=$(($count + 1))
[ $count -lt 10 ] || \
die "Recursion too deep in $infile - only 10 levels allowed!"
instring="$outstring"
done
# Now regular variables.
outstring=$(_substitute_vars "$instring" "@@")
[ $? -eq 0 ] || die "Failed to expand template $infile"
if [ -n "$outfile" ] ; then
echo "$outstring" > "$outfile"
else
echo "$outstring"
fi
)}
# Delimiter @@ means to substitute contents of variable.
# Delimiter @@@ means to substitute contents of file named by variable.
# @@@ supports leading '|' in variable value, which means to excute a
# command.
_substitute_vars() {(
instring="$1"
delimiter="${2:-@@}"
# get the list of variables used in the template
VARS=`echo "$instring" |
tr -cs "A-Z0-9_$delimiter" '\012' |
sort -u |
sed -n -e "s#^${delimiter}\(.*\)${delimiter}\\$#\1#p"`
tmp=$(mktemp)
for v in $VARS; do
# variable variables are fun .....
[ "${!v+x}" ] || {
rm -f $tmp
die "No substitution given for ${delimiter}$v${delimiter} in $infile"
}
s=${!v}
if [ "$delimiter" = "@@@" ] ; then
f=${s:-/dev/null}
c="${f#|}" # Is is a command, signified by a leading '|'?
if [ "$c" = "$f" ] ; then
# No leading '|', cat file.
s=$(cat -- "$f")
[ $? -eq 0 ] || {
rm -f $tmp
die "Could not substitute contents of file $f"
}
else
# Leading '|', execute command.
# Quoting problems here - using eval "$c" doesn't help.
s=$($c)
[ $? -eq 0 ] || {
rm -f $tmp
die "Could not execute command $c"
}
fi
fi
# escape some pesky chars
s=${s//
/\\n}
s=${s//#/\\#}
s=${s//&/\\&}
echo "s#${delimiter}${v}${delimiter}#${s}#g"
done > $tmp
echo "$instring" | sed -f $tmp
rm -f $tmp
)}
check_command() {
which $1 > /dev/null || die "Please install $1 to continue"
}
# Set a variable if it isn't already set. This allows environment
# variables to override default config settings.
defconf() {
local v="$1"
local e="$2"
[ "${!v+x}" ] || eval "$v=\"$e\""
}
load_config () {
local i
for i in "${installdir}/config.d/"*.defconf ; do
. "$i"
done
}
# Print the list of config variables defined in config.d/.
get_config_options () {( # sub-shell for local declaration of defconf()
local options=
defconf() { options="$options $1" ; }
load_config
echo $options
)}
# Produce a list of long options, suitable for use with getopt, that
# represent the config variables defined in config.d/.
getopt_config_options () {
local x=$(get_config_options | tr 'A-Z_' 'a-z-')
echo "${x// /:,}:"
}
# Unconditionally set the config variable associated with the given
# long option.
setconf_longopt () {
local longopt="$1"
local e="$2"
local v=$(echo "${longopt#--}" | tr 'a-z-' 'A-Z_')
# unset so defconf will set it
eval "unset $v"
defconf "$v" "$e"
}
# Dump all of the current config variables.
dump_config() {
local o
for o in $(get_config_options) ; do
echo "${o}=\"${!o}\""
done
exit 0
}
# $COLUMNS is set in interactive bash shells. It probably isn't set
# in this shell, so let's set it if it isn't.
: ${COLUMNS:=$(stty size 2>/dev/null | sed -e 's@.* @@')}
: ${COLUMNS:=80}
export COLUMNS
# Print text assuming it starts after other text in $startcol and
# needs to wrap before $COLUMNS - 2. Subsequent lines start at $startcol.
# Long "words" will extend past $COLUMNS - 2.
fill_text() {
local startcol="$1"
local text="$2"
local width=$(($COLUMNS - 2 - $startcol))
[ $width -lt 0 ] && width=$((78 - $startcol))
local out=""
local padding
if [ $startcol -gt 0 ] ; then
padding=$(printf "\n%${startcol}s" " ")
else
padding="
"
fi
while [ -n "$text" ] ; do
local orig="$text"
# If we already have output then arrange padding on the next line.
[ -n "$out" ] && out="${out}${padding}"
# Break the text at $width.
out="${out}${text:0:${width}}"
text="${text:${width}}"
# If we have left over text then the line break may be ugly,
# so let's check and try to break it on a space.
if [ -n "$text" ] ; then
# The 'x's stop us producing a special character like '(',
# ')' or '!'. Yuck - there must be a better way.
if [ "x${text:0:1}" != "x " -a "x${text: -1:1}" != "x " ] ; then
# We didn't break on a space. Arrange for the
# beginning of the broken "word" to appear on the next
# line but not if it will make us loop infinitely.
if [ "${orig}" != "${out##* }${text}" ] ; then
text="${out##* }${text}"
out="${out% *}"
else
# Hmmm, doing that would make us loop, so add the
# rest of the word from the remainder of the text
# to this line and let it extend past $COLUMNS - 2.
out="${out}${text%% *}"
if [ "${text# *}" != "$text" ] ; then
# Remember the text after the next space for next time.
text="${text# *}"
else
# No text after next space.
text=""
fi
fi
else
# We broke on a space. If it will be at the beginning
# of the next line then remove it.
text="${text# }"
fi
fi
done
echo "$out"
}
# Display usage text, trying these approaches in order.
# 1. See if it all fits on one line before $COLUMNS - 2.
# 2. See if splitting before the default value and indenting it
# to $startcol means that nothing passes $COLUMNS - 2.
# 3. Treat the message and default value as a string and just us fill_text()
# to format it.
usage_display_text () {
local startcol="$1"
local desc="$2"
local default="$3"
local width=$(($COLUMNS - 2 - $startcol))
[ $width -lt 0 ] && width=$((78 - $startcol))
default="(default \"$default\")"
if [ $((${#desc} + 1 + ${#default})) -le $width ] ; then
echo "${desc} ${default}"
else
local padding=$(printf "%${startcol}s" " ")
if [ ${#desc} -lt $width -a ${#default} -lt $width ] ; then
echo "$desc"
echo "${padding}${default}"
else
fill_text $startcol "${desc} ${default}"
fi
fi
}
# Display usage information for long config options.
usage_smart_display () {( # sub-shell for local declaration of defconf()
local startcol=33
defconf() {
local local longopt=$(echo "$1" | tr 'A-Z_' 'a-z-')
printf " --%-25s " "${longopt}=${3}"
usage_display_text $startcol "$4" "$2"
}
"$@"
)}
# Display usage information for long config options.
usage_config_options (){
usage_smart_display load_config
}
list_releases () {
local releases=$(cd $installdir/releases && echo *.release)
releases="${releases//.release}"
releases="${releases// /\", \"}"
echo "\"$releases\""
}
with_release () {
local release="$1"
# This simply loads an extra config file from $installdir/releases
f="${installdir}/releases/${release}.release"
if [ -r "$f" ] ; then
. "$f"
else
echo "Unknown release \"${release}\" specified to --with-release"
printf "%-25s" "Supported releases are: "
# The 70 is lazy but it will do.
fill_text 25 "$(list_releases)"
exit 1
fi
}
make_public_addresses () {
local firstip="${1:-$(($FIRSTIP + 1))}" # Default is $FIRSTIP + 1
local excluded_nodes="${2:-1}" # Comma separated, default = 1
local num_addrs="${3:-${NUMNODES}}" # Default is $NUMNODES
excluded_nodes=",${excluded_nodes}," # For delimiting matches.
local n e i
for n in $(seq 1 $num_addrs) ; do
echo "[/etc/ctdb/public_addresses:@@CLUSTER@@n${n}.@@DOMAIN@@]"
if [ "${excluded_nodes/,${n},}" = "$excluded_nodes" ] ; then
for e in "1" "2" ; do
for i in $(seq $firstip $(($firstip + $num_addrs - 1))) ; do
printf "\t@@IPBASE@@.${e}.${i}/24 eth${e}\n"
done
done
fi
echo
done
}
######################################################################
load_config
############################
# parse command line options
long_opts=$(getopt_config_options)
getopt_output=$(getopt -n autocluster -o "c:e:xh" -l help,dump,with-release: -l "$long_opts" -- "$@")
[ $? != 0 ] && usage
use_default_config=true
# We do 2 passes of the options. The first time we just handle usage
# and check whether -c is being used.
eval set -- "$getopt_output"
while true ; do
case "$1" in
-c) shift 2 ; use_default_config=false ;;
-e) shift 2 ;;
--) shift ; break ;;
--with-release) shift 2 ;; # Don't set use_default_config=false!!!
--dump|-x) shift ;;
-h|--help) usage ;; # Usage should be shown here for real defaults.
--*) shift 2 ;; # Assume other long opts are valid and take an arg.
*) usage ;; # shouldn't happen, so this is reasonable.
esac
done
config="./config"
$use_default_config && [ -r "$config" ] && . "$config"
eval set -- "$getopt_output"
while true ; do
case "$1" in
# force at least ./local_file to avoid accidental file from $PATH
-c) . "$(dirname $2)/$(basename $2)" ; shift 2 ;;
-e) eval "$2" ; exit ;;
--with-release)
with_release "$2"
shift 2
;;
-x) set -x; shift ;;
--dump) dump_config ;;
--) shift ; break ;;
-h|--help) usage ;; # Redundant.
--*)
# Putting --opt1|opt2|... into a variable and having case
# match against it as a pattern doesn't work. The | is
# part of shell syntax, so we need to do this. Look away
# now to stop your eyes from bleeding! :-)
x=",${long_opts}" # Now each option is surrounded by , and :
if [ "$x" != "${x#*,${1#--}:}" ] ; then
# Our option, $1, surrounded by , and : was in $x, so is legal.
setconf_longopt "$1" "$2"; shift 2
else
usage
fi
;;
*) usage ;; # shouldn't happen, so this is reasonable.
esac
done
# catch errors
set -e
set -E
trap 'es=$?;
echo ERROR: failed in function \"${FUNCNAME}\" at line ${LINENO} of ${BASH_SOURCE[0]} with code $es;
exit $es' ERR
# check for needed programs
check_command nbd-client
check_command expect
check_command $QEMU_NBD
[ $# -lt 1 ] && usage
command="$1"
shift
case $command in
create)
type=$1
shift
case $type in
base)
[ $# != 0 ] && usage
create_base;
;;
cluster)
[ $# != 1 ] && usage
create_cluster "$1";
;;
*)
usage;
;;
esac
;;
mount)
[ $# != 1 ] && usage
mount_disk "$1"
;;
unmount)
[ $# != 0 ] && usage
unmount_disk
;;
bootbase)
boot_base;
;;
testproxy)
test_proxy;
;;
*)
usage;
;;
esac