2 # main autocluster script
4 # Copyright (C) Andrew Tridgell 2008
5 # Copyright (C) Martin Schwenke 2008
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, see <http://www.gnu.org/licenses/>.
20 ##BEGIN-INSTALLDIR-MAGIC##
21 # There are better ways of doing this but not if you still want to be
22 # able to run straight out of a git tree. :-)
24 installdir="`dirname \"$0\"`"
26 autocluster=`which $0`
27 installdir="`dirname \"$autocluster\"`"
29 ##END-INSTALLDIR-MAGIC##
36 Usage: autocluster [OPTION] ... <COMMAND>
38 -c <file> specify config file (default is "config")
41 local releases=$(echo $(cd $installdir/releases && ls))
42 releases="\"${releases// /\", \"}\""
45 defconf "WITH_RELEASE" "" \
46 "<string>" "specify kickstart and yum templates using a release version string. Possible values are: ${releases}."
49 -x enable script debugging
50 --dump dump config settings and exit
52 configuration options:
63 create cluster CLUSTERNAME
66 create node CLUSTERNAME NODENUMBER
67 create one cluster node
69 create tsm CLUSTERNAME
70 create a TSM server node
73 mount a qemu disk on mnt/
76 unmount a qemu disk from mnt/
87 ###############################
88 # common node creation stuff
89 create_node_common() {
94 if [ "$node" = "tsm" ] ; then
97 template_file=$TSM_TEMPLATE
98 echo "Creating TSM cluster node $NAME"
101 NAME="${CLUSTER}n${nodenumber}"
102 template_file=$NODE_TEMPLATE
103 echo "Creating cluster node $NAME"
106 IPNUM=$(($FIRSTIP + $nodenumber))
107 DISK="${VIRTBASE}/${CLUSTER}/${NAME}.qcow2"
109 mkdir -p $VIRTBASE/$CLUSTER tmp
111 echo "Creating the disk"
113 qemu-img create -b "$VIRTBASE/$BASENAME.img" -f qcow2 "$DISK"
120 set_macaddrs $CLUSTER $nodenumber
123 echo "Creating $NAME.xml"
124 substitute_vars $template_file tmp/$NAME.xml
126 # install the XML file
127 $VIRSH undefine $NAME > /dev/null 2>&1 || true
128 $VIRSH define tmp/$NAME.xml
131 ###############################
132 # create a single node
135 local nodenumber="$2"
137 # first node might need more memory for SoFS GUI
138 if [ "$WITH_SOFS_GUI" = 1 -a $nodenumber = 1 -a $GUIMEM -gt $MEM ]; then
144 create_node_common "$nodenumber"
147 ###############################
152 # This must be before create_node_common.
153 TSMDISK="${VIRTBASE}/${CLUSTER}/tsmstorage.qcow2"
155 create_node_common "tsm"
157 echo "Creating tsm disk"
158 qemu-img create -f qcow2 "$TSMDISK" $TSMDISKSIZE
161 ###############################
162 # create a whole cluster
166 mkdir -p $VIRTBASE/$CLUSTER tmp
169 echo "Creating 3 shared disks"
170 for i in `seq 1 3`; do
171 # setup a nice ID at the start of the disk
172 qemu-img create -f raw $VIRTBASE/$CLUSTER/shared$i $SHAREDDISKSIZE
173 echo "SOFS-`uuidgen`" > tmp/diskid
174 dd if=tmp/diskid of=$VIRTBASE/$CLUSTER/shared$i conv=notrunc bs=1 > /dev/null 2>&1
177 echo "Creating $NUMNODES base nodes"
178 for i in `seq 1 $NUMNODES`; do
179 create_node "$CLUSTER" $i
182 if [ $WITH_TSM_NODE -eq 1 ]; then
183 echo "Creating TSM server node"
184 create_tsm "$CLUSTER"
187 echo "# autocluster $CLUSTER" > hosts.$CLUSTER
188 [ $WITH_TSM_NODE -eq 1 ] && {
189 echo "$IPBASE.0.$FIRSTIP ${CLUSTER}tsm.$LOWDOMAIN ${CLUSTER}tsm" >> hosts.$CLUSTER
191 for i in `seq 1 $NUMNODES`; do
192 echo "$IPBASE.0.`expr $FIRSTIP + $i` ${CLUSTER}n$i.$LOWDOMAIN ${CLUSTER}n$i" >> hosts.$CLUSTER
194 echo >> hosts.$CLUSTER
196 echo "Cluster $CLUSTER created"
197 echo "You may want to add this to your /etc/hosts file:"
203 ###############################
204 # test the proxy setup
206 export http_proxy=$WEBPROXY
207 wget -O /dev/null $INSTALL_SERVER || {
208 echo "Your WEBPROXY setting \"$WEBPROXY\" is not working"
219 DISK="$VIRTBASE/$NAME.img"
223 echo "Testing WEBPROXY $WEBPROXY"
226 echo "Creating the disk"
227 qemu-img create -f $BASE_FORMAT "$DISK" $DISKSIZE
230 mkdir -p mnt tmp tmp/ISO
235 echo "Creating kickstart file from template"
236 substitute_vars "$KICKSTART" "tmp/ks.cfg"
238 if [ $INSTALLKEY = "--skip" ]; then
240 --------------------------------------------------------------------------------------
241 WARNING: You have not entered an install key. Some RHEL packages will not be installed.
243 Please enter a valid RHEL install key in your config file like this:
245 INSTALLKEY="1234-5678-0123-4567"
247 The install will continue without an install key in 5 seconds
248 --------------------------------------------------------------------------------------
253 echo "Creating kickstart floppy"
254 dd if=/dev/zero of=tmp/floppy.img bs=1024 count=1440
255 mkdosfs tmp/floppy.img
256 mount -o loop -t msdos tmp/floppy.img mnt
258 mount -o loop,ro $ISO tmp/ISO
260 echo "Setting up bootloader"
261 cp tmp/ISO/isolinux/isolinux.bin tmp
262 cp tmp/ISO/isolinux/vmlinuz tmp
263 cp tmp/ISO/isolinux/initrd.img tmp
269 substitute_vars $INSTALL_TEMPLATE tmp/$NAME.xml
271 rm -f $KVMLOG/serial.$NAME
273 # boot the install CD
274 $VIRSH create tmp/$NAME.xml
276 echo "Waiting for install to start"
279 # wait for the install to finish
280 if ! waitfor $KVMLOG/serial.$NAME "you may safely reboot your system" 3600; then
282 echo "Failed to create base image $DISK"
291 Install finished, base image $DISK created
295 To ensure that this image does not change
297 Note that the root password has been set to $ROOTPASSWORD
302 ###############################
308 DISK="$VIRTBASE/$NAME.img"
322 echo "Creating $NAME.xml"
323 substitute_vars $BOOT_TEMPLATE tmp/$NAME.xml
325 # boot the base system
326 $VIRSH create tmp/$NAME.xml
329 ######################################################################
331 # various functions...
333 # Set some MAC address variables based on a hash of the cluster name
334 # plus the node number and each adapter number.
337 local nodenumber="$2"
339 local md5=$(echo $cname | md5sum)
340 local nh=$(printf "%02x" $nodenumber)
341 local mac_prefix="02:${md5:0:2}:${md5:2:2}:00:${nh}:"
343 MAC1="${mac_prefix}01"
344 MAC2="${mac_prefix}02"
345 MAC3="${mac_prefix}03"
346 MAC4="${mac_prefix}04"
347 MAC5="${mac_prefix}05"
348 MAC6="${mac_prefix}06"
351 # mount a qemu image via nbd
353 echo "Connecting nbd to $1"
356 killall -9 -q $QEMU_NBD || true
357 $QEMU_NBD -p 1300 $1 &
359 [ -r $NBD_DEVICE ] || {
360 mknod $NBD_DEVICE b 43 0
362 umount mnt 2> /dev/null || true
363 nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
364 killall -9 -q nbd-client || true
365 nbd-client localhost 1300 $NBD_DEVICE > /dev/null 2>&1 || true &
371 echo "Disconnecting nbd"
373 nbd-client -d $NBD_DEVICE > /dev/null 2>&1 || true
374 killall -9 -q nbd-client || true
375 killall -q $QEMU_NBD || true
378 # mount a qemu image via nbd
381 echo "Mounting disk $1"
383 for i in `seq 1 5`; do
384 mount -o offset=32256 $NBD_DEVICE mnt && {
388 umount mnt 2>/dev/null || true
391 [ $mount_ok = 1 ] || {
392 echo "Failed to mount $1"
396 echo "Mounted directory does not look like a root filesystem"
402 # unmount a qemu image
404 echo "Unmounting disk"
406 umount mnt || umount mnt || true
410 # setup the files from $BASE_TEMPLATES/, substituting any variables
411 # based on the config
415 echo "Copy base files"
416 for f in `cd $BASE_TEMPLATES && find . \! -name '*~'`; do
417 if [ -d "$BASE_TEMPLATES/$f" ]; then
420 substitute_vars "$BASE_TEMPLATES/$f" "mnt/$f"
422 chmod --reference="$BASE_TEMPLATES/$f" "mnt/$f"
424 # this is needed as git doesn't store file permissions other
426 chmod 600 mnt/etc/ssh/*key mnt/root/.ssh/*
427 chmod 700 mnt/etc/ssh mnt/root/.ssh mnt/root
428 if [ -r "$HOME/.ssh/id_rsa.pub" ]; then
429 echo "Adding $HOME/.ssh/id_rsa.pub to ssh authorized_keys"
430 cat "$HOME/.ssh/id_rsa.pub" >> mnt/root/.ssh/authorized_keys
432 if [ -r "$HOME/.ssh/id_dsa.pub" ]; then
433 echo "Adding $HOME/.ssh/id_dsa.pub to ssh authorized_keys"
434 cat "$HOME/.ssh/id_dsa.pub" >> mnt/root/.ssh/authorized_keys
436 echo "Adjusting grub.conf"
437 local o="$EXTRA_KERNEL_OPTIONS"
438 sed -e "s/console=ttyS0,19200/console=ttyS0,115200/" \
439 -e "s/ nodmraid//" -e "s/ nompath//" \
440 -e "s/quiet/divider=10${o:+ }${o}/g" mnt/boot/grub/grub.conf -i.org
444 # Order is very important here, since postinstall.sh can use
447 substitute_vars $YUM_TEMPLATE tmp/SOFS.repo
448 YUM_REPOS="`cat tmp/SOFS.repo`"
450 substitute_vars $POSTINSTALL_TEMPLATE tmp/postinstall.sh
451 EXTRA_POSTINSTALL="`cat tmp/postinstall.sh`"
453 substitute_vars "$installdir/templates/force-net.sh" tmp/force-net.sh
454 AUTOCLUSTER_POSTINSTALL="`cat tmp/force-net.sh`"
457 # setup various networking components
459 echo "Setting up networks"
461 for i in `seq 1 $NUMNODES`; do
462 echo "$IPBASE.0.`expr $FIRSTIP + $i` ${CLUSTER}n$i.$LOWDOMAIN ${CLUSTER}n$i" >> mnt/etc/hosts
465 echo "Setting up /etc/ctdb/nodes"
466 mkdir -p mnt/etc/ctdb
467 rm -f mnt/etc/ctdb/nodes
468 for i in `seq 1 $NUMNODES`; do
469 echo "$IPBASE.0.`expr $FIRSTIP + $i`" >> mnt/etc/ctdb/nodes
472 [ "$WEBPROXY" = "" ] || {
473 echo "export http_proxy=$WEBPROXY" >> mnt/etc/bashrc
476 echo "Enabling nfs mount of $NFSSHARE"
477 mkdir -p mnt/root/SOFS
478 echo "$NFSSHARE /root/SOFS nfs intr" >> mnt/etc/fstab
480 mkdir -p mnt/etc/yum.repos.d
481 substitute_vars "$YUM_TEMPLATE" mnt/etc/yum.repos.d/SOFS.repo
485 [ -z "$TIMEZONE" ] && {
486 [ -r /etc/timezone ] && {
487 TIMEZONE=`cat /etc/timezone`
489 [ -r /etc/sysconfig/clock ] && {
490 . /etc/sysconfig/clock
494 if [ "$TIMEZONE" = "" ]; then
495 echo "Unable to determine TIMEZONE - please set in config"
500 # substite a set of variables of the form @@XX@@ for the shell
501 # variables $XX in a file
506 delim2="$delimiter$delimiter"
508 # get the list of variables used in the template
509 VARS=`tr -cs "A-Z0-9_$delimiter" '\012' < "$infile" |
511 egrep "^$delim2.*$delim2$" |
512 cut -d$delimiter -f3`
517 # variable variables are fun .....
519 echo "ERROR: No substitution given for ${delim2}$v${delim2} in $infile"
524 # escape some pesky chars
529 echo "s#@@$v@@#$s#g" >> sed.$$
532 sed -f sed.$$ < "$infile" > "$outfile" || exit 1
538 if which $1 > /dev/null; then
539 # echo "$1 is installed: OK"
542 echo "Please install $1 to continue"
547 # Set a variable if it isn't already set. This allows environment
548 # variables to override default config settings.
553 [ "${!v+x}" ] || eval "$v=\"$e\""
556 # Print the list of config variables defined in config.default.
557 get_config_options() {( # sub-shell for local declaration of defconf()
559 defconf() { options="$options $1" ; }
560 . "$installdir/config.default"
564 # Produce a list of long options, suitable for use with getopt, that
565 # represent the config variables defined in config.default.
566 getopt_config_options() {
567 local x=$(get_config_options | tr 'A-Z_' 'a-z-')
571 # Unconditionally set the config variable associated with the given
577 local v=$(echo "${longopt#--}" | tr 'a-z-' 'A-Z_')
578 # unset so defconf will set it
583 # Dump all of the current config variables.
586 for o in $(get_config_options) ; do
587 echo "${o}=\"${!o}\""
592 # Print text assuming it starts after other text in $startcol and
593 # needs to wrap before $fillcol. Subsequent lines start at $startcol.
594 # Long "words" will extend past $fillcol.
600 local width=$(($fillcol - $startcol))
601 [ $width -lt 0 ] && width=$((78 - $startcol))
605 local padding=$(printf "\n%${startcol}s" " ")
607 while [ -n "$text" ] ; do
610 # If we already have output then arrange padding on the next line.
611 [ -n "$out" ] && out="${out}${padding}"
613 # Break the text at $width.
614 out="${out}${text:0:${width}}"
615 text="${text:${width}}"
617 # If we have left over text then the line break may be ugly,
618 # so let's check and try to break it on a space.
619 if [ -n "$text" ] ; then
620 if [ "${text:0:1}" != " " -a "${text: -1:1}" != " " ] ; then
621 # We didn't break on a space. Arrange for the
622 # beginning of the broken "word" to appear on the next
623 # line but not if it will make us loop infinitely.
624 if [ "${orig}" != "${out##* }${text}" ] ; then
625 text="${out##* }${text}"
628 # Hmmm, doing that would make us loop, so add the
629 # rest of the word from the remainder of the text
630 # to this line and let it extend past $fillcol.
631 out="${out}${text%% *}"
632 if [ "${text# *}" != "$text" ] ; then
633 # Remember the text after the next space for next time.
636 # No text after next space.
641 # We broke on a space. If it will be at the beginning
642 # of the next line then remove it.
651 # Display usage text, trying these approaches in order.
652 # 1. See if it all fits on one line before $fillcol.
653 # 2. See if splitting before the default value and indenting it
654 # to $startcol means that nothing passes $fillcol.
655 # 3. Treat the message and default value as a string and just us fill_text()
657 usage_display_text() {
663 local width=$(($fillcol - $startcol))
665 # To avoid clutter, only quote values that need it.
667 #[ -z "$default" -o "$(echo $default)" = "${default// /}" ] || q="\""
668 default="(default \"$default\")"
670 if [ $((${#desc} + 1 + ${#default})) -le $width ] ; then
671 echo "${desc} ${default}"
673 local padding=$(printf "%${startcol}s" " ")
675 if [ ${#desc} -lt $width -a ${#default} -lt $width ] ; then
677 echo "${padding}${default}"
679 fill_text $startcol $fillcol "${desc} ${default}"
684 # Display usage information for long config options.
685 usage_smart_display () {( # sub-shell for local declaration of defconf()
687 local fillcol=$def_fillcol
688 local rows=$(stty size | sed -e 's@.* @@')
689 [ -n "$rows" ] && fillcol=$(($rows - 2))
693 # We need to have at least one column... negative is also a problem.
694 [ $fillcol -le $startcol ] && fillcol=$def_fillcol
697 local local longopt=$(echo "$1" | tr 'A-Z_' 'a-z-')
699 printf " --%-25s " "${longopt}=${3}"
701 usage_display_text $startcol $fillcol "$4" "$2"
708 # Display usage information for long config options.
709 usage_config_options(){
710 usage_smart_display . "$installdir/config.default"
713 ######################################################################
715 . "$installdir/config.default"
717 ############################
718 # parse command line options
719 long_opts=$(getopt_config_options)
720 getopt_output=$(getopt -n autocluster -o "c:xh" -l help,dump,with-release -l "$long_opts" -- "$@")
723 use_default_config=true
725 # We do 2 passes of the options. The first time we just handle usage
726 # and check whether -c is being used.
727 eval set -- "$getopt_output"
730 -c) shift 2 ; use_default_config=false ;;
732 --with-release) shift 2 ;; # Don't set use_default_config=false!!!
734 -h|--help) usage ;; # Usage should be shown here for real defaults.
735 --*) shift 2 ;; # Assume other long opts are valid and take an arg.
736 *) usage ;; # shouldn't happen, so this is reasonable.
741 $use_default_config && [ -r "$config" ] && . "$config"
743 eval set -- "$getopt_output"
747 -c) . "`dirname $2`/$2" ; shift 2 ;;
749 # This simply loads an extra config file from $installdir/releases
750 f="${installdir}/releases/$2"
751 if [ -r "$f" ] ; then
754 echo "Unknown release \"$2\" specified to --with-release"
760 --dump) dump_config ;;
762 -h|--help) usage ;; # Redundant.
764 # Putting --opt1|opt2|... into a variable and having case
765 # match against it as a pattern doesn't work. The | is
766 # part of shell syntax, so we need to do this. Look away
767 # now to stop your eyes from bleeding! :-)
768 x=",${long_opts}" # Now each option is surrounded by , and :
769 if [ "$x" != "${x#*,${1#--}:}" ] ; then
770 # Our option, $1, surrounded by , and : was in $x, so is legal.
771 setconf_longopt "$1" "$2"; shift 2
776 *) usage ;; # shouldn't happen, so this is reasonable.
784 echo ERROR: failed in function \"${FUNCNAME}\" at line ${LINENO} of ${BASH_SOURCE[0]} with code $es;
787 LOWDOMAIN=`echo $DOMAIN | tr A-Z a-z`
789 # check for needed programs
790 check_command nbd-client
792 check_command $QEMU_NBD
794 [ $# -lt 1 ] && usage
814 create_node "$1" "$2";