autocluster: Only retry "create" and "setup" steps during "build"

"create" and "setup" are used during development when the failures are
usually programming errors rather than intermittent errors, so don't
retry when those steps are run separately.

Signed-off-by: Martin Schwenke <martin@meltin.net>

diff --git a/autocluster.py b/autocluster.py
index 34342f86d5d982e55696dd62ec1dd950eef4defa..ed97f49643b235acec5234aee84f827edab8994c 100755 (executable)
--- a/autocluster.py
+++ b/autocluster.py
@@ -593,13 +593,13 @@ def create_shared_disk_images(cluster, config):
     conn.close()
 
 
-def cluster_destroy_quiet(cluster):
+def cluster_destroy_quiet(cluster, retries=1):
     '''Destroy and undefine cluster using Vagrant - don't announce'''
 
     config = load_config(cluster)
 
     # First attempt often fails, so try a few times
-    for _ in range(10):
+    for _ in range(retries):
         try:
             vagrant_command(cluster,
                             config,
@@ -613,7 +613,7 @@ def cluster_destroy_quiet(cluster):
     raise saved_err
 
 
-def cluster_destroy(cluster):
+def cluster_destroy(cluster, retries=1):
     '''Destroy and undefine cluster using Vagrant'''
 
     announce('cluster', cluster, 'destroy')
@@ -622,10 +622,10 @@ def cluster_destroy(cluster):
     if not os.path.exists(config_file):
         sys.exit('ERROR: Generated configuration for cluster does not exist')
 
-    cluster_destroy_quiet(cluster)
+    cluster_destroy_quiet(cluster, retries)
 
 
-def cluster_create(cluster):
+def cluster_create(cluster, retries=1):
     '''Create and boot cluster using Vagrant'''
 
     announce('cluster', cluster, 'create')
@@ -637,7 +637,7 @@ def cluster_create(cluster):
     create_shared_disk_images(cluster, config)
 
     # First attempt sometimes fails, so try a few times
-    for _ in range(10):
+    for _ in range(retries):
         try:
             vagrant_command(cluster, config, ['up'])
         except subprocess.CalledProcessError as err:
@@ -661,7 +661,7 @@ def cluster_ssh_config(cluster):
     shutil.copyfile(src, dst)
 
 
-def cluster_setup(cluster):
+def cluster_setup(cluster, retries=1):
     '''Setup cluster using Ansible'''
 
     announce('cluster', cluster, 'setup')
@@ -679,7 +679,7 @@ def cluster_setup(cluster):
             playbook]
 
     # First attempt sometimes fails, so try a few times
-    for _ in range(5):
+    for _ in range(retries):
         try:
             subprocess.check_call(args)
         except subprocess.CalledProcessError as err:
@@ -699,11 +699,11 @@ def cluster_build(cluster):
 
     config_file = get_config_file_path(cluster)
     if os.path.exists(config_file):
-        cluster_destroy(cluster)
+        cluster_destroy(cluster, 10)
     cluster_generate(cluster)
-    cluster_create(cluster)
+    cluster_create(cluster, 10)
     cluster_ssh_config(cluster)
-    cluster_setup(cluster)
+    cluster_setup(cluster, 5)
 
 
 def cluster_command(cluster, command):
@@ -718,7 +718,7 @@ def cluster_command(cluster, command):
     elif command == 'generate':
         cluster_generate(cluster)
     elif command == 'destroy':
-        cluster_destroy(cluster)
+        cluster_destroy(cluster, 10)
     elif command == 'create':
         cluster_create(cluster)
     elif command == 'ssh_config':