2 # main autocluster script
4 # Copyright (C) Andrew Tridgell 2008
5 # Copyright (C) Martin Schwenke 2008
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, see <http://www.gnu.org/licenses/>.
21 installdir="`dirname \"$0\"`"
23 autocluster=`which $0`
24 installdir="`dirname \"$autocluster\"`"
32 Usage: autocluster [OPTION] ... <COMMAND>
34 -c <file> specify config file (default is "config")
35 -x enable script debugging
36 --dump dump config settings and exit
38 configuration options:
49 create cluster CLUSTERNAME
52 create node CLUSTERNAME NODENUMBER
53 create one cluster node
55 create tsm CLUSTERNAME
56 create a TSM server node
59 mount a qemu disk on mnt/
62 unmount a qemu disk from mnt/
73 ###############################
74 # create a single node
78 NAME="$CLUSTER""n$NODENUMBER"
79 BASE="$VIRTBASE/$BASENAME.img"
80 DISK="$VIRTBASE/$CLUSTER/$NAME.qcow2"
83 # first node might need more memory for SoFS GUI
84 if [ "$WITH_SOFS_GUI" = 1 -a $NODENUMBER = 1 -a $GUIMEM -gt $MEM ]; then
89 echo "Creating cluster node $NAME"
91 mkdir -p $VIRTBASE/$CLUSTER
93 echo "Creating the disk"
95 qemu-img create -b "$BASE" -f qcow2 "$DISK"
97 IPNUM=`expr $FIRSTIP + $NODENUMBER`
106 MAC1=`get_macaddr $CLUSTER $NODENUMBER 1`
107 MAC2=`get_macaddr $CLUSTER $NODENUMBER 2`
108 MAC3=`get_macaddr $CLUSTER $NODENUMBER 3`
111 echo "Creating $NAME.xml"
112 substitute_vars $NODE_TEMPLATE tmp/$NAME.xml
114 # install the XML file
115 $VIRSH undefine $NAME > /dev/null 2>&1 || true
116 $VIRSH define tmp/$NAME.xml
119 ###############################
124 BASE="$VIRTBASE/$BASENAME.img"
125 DISK="$VIRTBASE/$CLUSTER/$NAME.qcow2"
126 TSMDISK="$VIRTBASE/$CLUSTER/tsmstorage.qcow2"
128 echo "Creating TSM cluster node $NAME"
130 mkdir -p $VIRTBASE/$CLUSTER tmp
132 echo "Creating the disk"
134 qemu-img create -b "$BASE" -f qcow2 "$DISK"
136 echo "Creating tsm disk"
137 qemu-img create -f qcow2 "$TSMDISK" $TSMDISKSIZE
139 # TSM server is first IP in the cluster
148 MAC1=`get_macaddr $CLUSTER 0 0`
150 echo "Creating $NAME.xml"
151 substitute_vars $TSM_TEMPLATE tmp/$NAME.xml
153 # install the XML file
154 $VIRSH undefine $NAME > /dev/null 2>&1 || true
155 $VIRSH define tmp/$NAME.xml
158 ###############################
159 # create a whole cluster
163 mkdir -p $VIRTBASE/$CLUSTER tmp
166 echo "Creating 3 shared disks"
167 for i in `seq 1 3`; do
168 # setup a nice ID at the start of the disk
169 qemu-img create -f raw $VIRTBASE/$CLUSTER/shared$i $SHAREDDISKSIZE
170 echo "SOFS-`uuidgen`" > tmp/diskid
171 dd if=tmp/diskid of=$VIRTBASE/$CLUSTER/shared$i conv=notrunc bs=1 > /dev/null 2>&1
174 echo "Creating $NUMNODES base nodes"
175 for i in `seq 1 $NUMNODES`; do
176 create_node "$CLUSTER" $i
179 if [ $WITH_TSM_NODE -eq 1 ]; then
180 echo "Creating TSM server node"
181 create_tsm "$CLUSTER"
184 echo "# autocluster $CLUSTER" > hosts.$CLUSTER
185 [ $WITH_TSM_NODE -eq 1 ] && {
186 echo "$IPBASE.0.$FIRSTIP ${CLUSTER}tsm.$LOWDOMAIN ${CLUSTER}tsm" >> hosts.$CLUSTER
188 for i in `seq 1 $NUMNODES`; do
189 echo "$IPBASE.0.`expr $FIRSTIP + $i` ${CLUSTER}n$i.$LOWDOMAIN ${CLUSTER}n$i" >> hosts.$CLUSTER
191 echo >> hosts.$CLUSTER
193 echo "Cluster $CLUSTER created"
194 echo "You may want to add this to your /etc/hosts file:"
200 ###############################
201 # test the proxy setup
203 export http_proxy=$WEBPROXY
204 wget -O /dev/null $INSTALL_SERVER || {
205 echo "Your WEBPROXY setting \"$WEBPROXY\" is not working"
216 DISK="$VIRTBASE/$NAME.img"
220 echo "Testing WEBPROXY $WEBPROXY"
223 echo "Creating the disk"
224 qemu-img create -f $BASE_FORMAT "$DISK" $DISKSIZE
227 mkdir -p mnt tmp tmp/ISO
232 echo "Creating kickstart file from template"
233 substitute_vars "$KICKSTART" "tmp/ks.cfg"
235 if [ $INSTALLKEY = "--skip" ]; then
237 --------------------------------------------------------------------------------------
238 WARNING: You have not entered an install key. Some RHEL packages will not be installed.
240 Please enter a valid RHEL install key in your config file like this:
242 INSTALLKEY="1234-5678-0123-4567"
244 The install will continue without an install key in 5 seconds
245 --------------------------------------------------------------------------------------
250 echo "Creating kickstart floppy"
251 dd if=/dev/zero of=tmp/floppy.img bs=1024 count=1440
252 mkdosfs tmp/floppy.img
253 mount -o loop -t msdos tmp/floppy.img mnt
255 mount -o loop,ro $ISO tmp/ISO
257 echo "Setting up bootloader"
258 cp tmp/ISO/isolinux/isolinux.bin tmp
259 cp tmp/ISO/isolinux/vmlinuz tmp
260 cp tmp/ISO/isolinux/initrd.img tmp
266 substitute_vars $INSTALL_TEMPLATE tmp/$NAME.xml
268 rm -f $KVMLOG/serial.$NAME
270 # boot the install CD
271 $VIRSH create tmp/$NAME.xml
273 echo "Waiting for install to start"
276 # wait for the install to finish
277 if ! waitfor $KVMLOG/serial.$NAME "you may safely reboot your system" 3600; then
279 echo "Failed to create base image $DISK"
288 Install finished, base image $DISK created
292 To ensure that this image does not change
294 Note that the root password has been set to $ROOTPASSWORD
299 ###############################
305 DISK="$VIRTBASE/$NAME.img"
319 echo "Creating $NAME.xml"
320 substitute_vars $BOOT_TEMPLATE tmp/$NAME.xml
322 # boot the base system
323 $VIRSH create tmp/$NAME.xml
326 ######################################################################
328 # various functions...
330 # create a MAC address based on a hash of the cluster name
331 # plus the adapter and node number
337 oct2="`echo $cname | md5sum | cut -c1-2`"
338 oct3="`echo $cname | md5sum | cut -c3-4`"
342 echo "${oct1}:${oct2}:${oct3}:${oct4}:${oct5}:${oct6}"
345 # mount a qemu image via nbd
347 echo "Connecting nbd to $1"
350 killall -9 -q $QEMU_NBD || true
351 $QEMU_NBD -p 1300 $1 &
353 [ -r $NBD_DEVICE ] || {
354 mknod $NBD_DEVICE b 43 0
356 umount mnt 2> /dev/null || true
357 nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
358 killall -9 -q nbd-client || true
359 nbd-client localhost 1300 $NBD_DEVICE > /dev/null 2>&1 || true &
365 echo "Disconnecting nbd"
367 nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
368 killall -9 -q nbd-client || true
369 killall -q $QEMU_NBD || true
372 # mount a qemu image via nbd
375 echo "Mounting disk $1"
377 for i in `seq 1 5`; do
378 mount -o offset=32256 $NBD_DEVICE mnt && {
382 umount mnt 2>/dev/null || true
385 [ $mount_ok = 1 ] || {
386 echo "Failed to mount $1"
390 echo "Mounted directory does not look like a root filesystem"
396 # unmount a qemu image
398 echo "Unmounting disk"
400 umount mnt || umount mnt || true
404 # setup the files from $BASE_TEMPLATES/, substituting any variables
405 # based on the config
409 echo "Copy base files"
410 for f in `cd $BASE_TEMPLATES && find . \! -name '*~'`; do
411 if [ -d "$BASE_TEMPLATES/$f" ]; then
414 substitute_vars "$BASE_TEMPLATES/$f" "mnt/$f"
416 chmod --reference="$BASE_TEMPLATES/$f" "mnt/$f"
418 # this is needed as git doesn't store file permissions other
420 chmod 600 mnt/etc/ssh/*key mnt/root/.ssh/*
421 chmod 700 mnt/etc/ssh mnt/root/.ssh mnt/root
422 if [ -r "$HOME/.ssh/id_rsa.pub" ]; then
423 echo "Adding $HOME/.ssh/id_rsa.pub to ssh authorized_keys"
424 cat "$HOME/.ssh/id_rsa.pub" >> mnt/root/.ssh/authorized_keys
426 if [ -r "$HOME/.ssh/id_dsa.pub" ]; then
427 echo "Adding $HOME/.ssh/id_dsa.pub to ssh authorized_keys"
428 cat "$HOME/.ssh/id_dsa.pub" >> mnt/root/.ssh/authorized_keys
430 echo "Adjusting grub.conf"
431 local o="$EXTRA_KERNEL_OPTIONS"
432 sed -e "s/console=ttyS0,19200/console=ttyS0,115200/" \
433 -e "s/ nodmraid//" -e "s/ nompath//" \
434 -e "s/quiet/divider=10${o:+ }${o}/g" mnt/boot/grub/grub.conf -i.org
438 # Order is very important here, since postinstall.sh can use
441 substitute_vars $YUM_TEMPLATE tmp/SOFS.repo
442 YUM_REPOS="`cat tmp/SOFS.repo`"
444 substitute_vars $POSTINSTALL_TEMPLATE tmp/postinstall.sh
445 EXTRA_POSTINSTALL="`cat tmp/postinstall.sh`"
447 substitute_vars "$installdir/templates/force-net.sh" tmp/force-net.sh
448 AUTOCLUSTER_POSTINSTALL="`cat tmp/force-net.sh`"
451 # setup various networking components
453 echo "Setting up networks"
455 for i in `seq 1 $NUMNODES`; do
456 echo "$IPBASE.0.`expr $FIRSTIP + $i` ${CLUSTER}n$i.$LOWDOMAIN ${CLUSTER}n$i" >> mnt/etc/hosts
459 echo "Setting up /etc/ctdb/nodes"
460 mkdir -p mnt/etc/ctdb
461 rm -f mnt/etc/ctdb/nodes
462 for i in `seq 1 $NUMNODES`; do
463 echo "$IPBASE.0.`expr $FIRSTIP + $i`" >> mnt/etc/ctdb/nodes
466 [ "$WEBPROXY" = "" ] || {
467 echo "export http_proxy=$WEBPROXY" >> mnt/etc/bashrc
470 echo "Enabling nfs mount of $NFSSHARE"
471 mkdir -p mnt/root/SOFS
472 echo "$NFSSHARE /root/SOFS nfs intr" >> mnt/etc/fstab
474 mkdir -p mnt/etc/yum.repos.d
475 substitute_vars "$YUM_TEMPLATE" mnt/etc/yum.repos.d/SOFS.repo
479 [ -z "$TIMEZONE" ] && {
480 [ -r /etc/timezone ] && {
481 TIMEZONE=`cat /etc/timezone`
483 [ -r /etc/sysconfig/clock ] && {
484 . /etc/sysconfig/clock
488 if [ "$TIMEZONE" = "" ]; then
489 echo "Unable to determine TIMEZONE - please set in config"
494 # substite a set of variables of the form @@XX@@ for the shell
495 # variables $XX in a file
500 delim2="$delimiter$delimiter"
502 # get the list of variables used in the template
503 VARS=`tr -cs "A-Z0-9_$delimiter" '\012' < "$infile" |
505 egrep "^$delim2.*$delim2$" |
506 cut -d$delimiter -f3`
511 # variable variables are fun .....
513 echo "ERROR: No substitution given for ${delim2}$v${delim2} in $infile"
518 # escape some pesky chars
523 echo "s#@@$v@@#$s#g" >> sed.$$
526 sed -f sed.$$ < "$infile" > "$outfile" || exit 1
532 if which $1 > /dev/null; then
533 # echo "$1 is installed: OK"
536 echo "Please install $1 to continue"
541 # Set a variable if it isn't already set. This allows environment
542 # variables to override default config settings.
547 [ "${!v+x}" ] || eval "$v=\"$e\""
550 # Print the list of config variables defined in config.default.
551 get_config_options() {( # sub-shell for local declaration of defconf()
553 defconf() { options="$options $1" ; }
554 . "$installdir/config.default"
558 # Produce a list of long options, suitable for use with getopt, that
559 # represent the config variables defined in config.default.
560 getopt_config_options() {
561 local x=$(get_config_options | tr 'A-Z_' 'a-z-')
565 # Unconditionally set the config variable associated with the given
571 local v=$(echo "${longopt#--}" | tr 'a-z-' 'A-Z_')
572 # unset so defconf will set it
577 # Dump all of the current config variables.
580 for o in $(get_config_options) ; do
581 echo "${o}=\"${!o}\""
586 # Print text assuming it starts after other text in $startcol and
587 # needs to wrap before $fillcol. Subsequent lines start at $startcol.
588 # Long "words" will extend past $fillcol.
594 local width=$(($fillcol - $startcol))
595 [ $width -lt 0 ] && width=$((78 - $startcol))
599 local padding=$(printf "\n%${startcol}s" " ")
601 while [ -n "$text" ] ; do
604 # If we already have output then arrange padding on the next line.
605 [ -n "$out" ] && out="${out}${padding}"
607 # Break the text at $width.
608 out="${out}${text:0:${width}}"
609 text="${text:${width}}"
611 # If we have left over text then the line break may be ugly,
612 # so let's check and try to break it on a space.
613 if [ -n "$text" ] ; then
614 if [ "${text:0:1}" != " " -a "${text: -1:1}" != " " ] ; then
615 # We didn't break on a space. Arrange for the
616 # beginning of the broken "word" to appear on the next
617 # line but not if it will make us loop infinitely.
618 if [ "${orig}" != "${out##* }${text}" ] ; then
619 text="${out##* }${text}"
622 # Hmmm, doing that would make us loop, so add the
623 # rest of the word from the remainder of the text
624 # to this line and let it extend past $fillcol.
625 out="${out}${text%% *}"
626 if [ "${text# *}" != "$text" ] ; then
627 # Remember the text after the next space for next time.
630 # No text after next space.
635 # We broke on a space. If it will be at the beginning
636 # of the next line then remove it.
645 # Display usage text, trying these approaches in order.
646 # 1. See if it all fits on one line before $fillcol.
647 # 2. See if splitting before the default value and indenting it
648 # to $startcol means that nothing passes $fillcol.
649 # 3. Treat the message and default value as a string and just us fill_text()
651 usage_display_text() {
657 local width=$(($fillcol - $startcol))
659 # To avoid clutter, only quote values that need it.
661 #[ -z "$default" -o "$(echo $default)" = "${default// /}" ] || q="\""
662 default="(default \"$default\")"
664 if [ $((${#desc} + 1 + ${#default})) -le $width ] ; then
665 echo "${desc} ${default}"
667 local padding=$(printf "%${startcol}s" " ")
669 if [ ${#desc} -lt $width -a ${#default} -lt $width ] ; then
671 echo "${padding}${default}"
673 fill_text $startcol $fillcol "${desc} ${default}"
678 # Display usage information for long config options.
679 usage_config_options(){( # sub-shell for local declaration of defconf()
681 local fillcol=$def_fillcol
682 local rows=$(stty size | sed -e 's@.* @@')
683 [ -n "$rows" ] && fillcol=$(($rows - 2))
687 # We need to have at least one column... negative is also a problem.
688 [ $fillcol -le $startcol ] && fillcol=$def_fillcol
691 local local longopt=$(echo "$1" | tr 'A-Z_' 'a-z-')
693 printf " --%-25s " "${longopt}=${3}"
695 usage_display_text $startcol $fillcol "$4" "$2"
697 . "$installdir/config.default"
701 ######################################################################
703 . "$installdir/config.default"
705 ############################
706 # parse command line options
707 long_opts=$(getopt_config_options)
708 getopt_output=$(getopt -n autocluster -o "c:xh" -l help,dump -l "$long_opts" -- "$@")
711 use_default_config=true
713 # We do 2 passes of the options. The first time we just handle usage
714 # and check whether -c is being used.
715 eval set -- "$getopt_output"
718 -c) shift 2 ; use_default_config=false ;;
721 -h|--help) usage ;; # Usage should be shown here for real defaults.
722 --*) shift 2 ;; # Assume other long opts are valid and take an arg.
723 *) usage ;; # shouldn't happen, so this is reasonable.
728 $use_default_config && [ -r "$config" ] && . "$config"
730 eval set -- "$getopt_output"
734 -c) . "`dirname $2`/$2" ; shift 2 ; conf_done=true ;;
736 --dump) dump_config ;;
738 -h|--help) usage ;; # Redundant.
740 # Putting --opt1|opt2|... into a variable and having case
741 # match against it as a pattern doesn't work. The | is
742 # part of shell syntax, so we need to do this. Look away
743 # now to stop your eyes from bleeding! :-)
744 x=",${long_opts}" # Now each option is surrounded by , and :
745 if [ "$x" != "${x#*,${1#--}:}" ] ; then
746 # Our option, $1, surrounded by , and : was in $x, so is legal.
747 setconf_longopt "$1" "$2"; shift 2
752 *) usage ;; # shouldn't happen, so this is reasonable.
760 echo ERROR: failed in function \"${FUNCNAME}\" at line ${LINENO} of ${BASH_SOURCE[0]} with code $es;
763 LOWDOMAIN=`echo $DOMAIN | tr A-Z a-z`
765 # check for needed programs
766 check_command nbd-client
768 check_command $QEMU_NBD
770 [ $# -lt 1 ] && usage
790 create_node "$1" "$2";