CI: split out "samba-ad-dc-ntvfs[-py2]" test targets

Many AD tests currently use the "samba" target. Split out a new target
"samba-ad-dc-ntvfs" and have all tests that use the "ad_dc_ntvfs" env
use the new target. This should greatly speed up the runtime for the "samba"
target and avoid swapping.

This reduces the total CI time by ~ 55%, I got an autobuild and a gitlab
pipeline finished in just ~ 100 mins!

Pair-Programmed-With: Stefan Metzmacher <metze@samba.org>

Signed-off-by: Ralph Boehme <slow@samba.org>
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Ralph Böhme <slow@samba.org>
Autobuild-Date(master): Mon Feb 11 14:10:12 CET 2019 on sn-devel-144

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 908c29ec9d95de5c88bacc6bf8ab09ba2e214944..0729599e1b87488d6c5fce85292ac3ff7a6710ae 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -118,3 +118,13 @@ build_samba_buildpy2_only:
   script:
     - python script/autobuild.py samba-buildpy2-only             --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase
 
+build_samba_ad_dc_ntvfs:
+  <<: *shared_template
+  script:
+    # this one takes about 100 mins to finish
+    - script/autobuild.py samba-ad-dc-ntvfs --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase
+
+build_samba_ad_dc_ntvfs_py2:
+  <<: *shared_template
+  script:
+    - script/autobuild.py samba-ad-dc-ntvfs-py2 --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase

diff --git a/script/autobuild.py b/script/autobuild.py
index 00f0d2202a3d8b40741c37990afbe61f976c4b18..b1bd9d1686c025d7847aa1528d977be3400950d2 100755 (executable)
--- a/script/autobuild.py
+++ b/script/autobuild.py
@@ -49,6 +49,8 @@ builddirs = {
     "samba-none-env": ".",
     "samba-ad-dc": ".",
     "samba-ad-dc-py2": ".",
+    "samba-ad-dc-ntvfs": ".",
+    "samba-ad-dc-ntvfs-py2": ".",
     "samba-ad-dc-2": ".",
     "samba-ad-dc-2-py2": ".",
     "samba-ad-dc-backup": ".",
@@ -93,8 +95,10 @@ tasks = {
                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
                ("clean", "make clean", "text/plain")],
 
-    # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
-    "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
+    # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+    "samba": [
+                ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                 ("make", "make -j", "text/plain"),
                 ("test", "make test FAIL_IMMEDIATELY=1 "
                  "TESTS='${PY3_ONLY}"
@@ -102,6 +106,7 @@ tasks = {
                  "--exclude-env=nt4_dc "
                  "--exclude-env=nt4_member "
                  "--exclude-env=ad_dc "
+                 "--exclude-env=ad_dc_ntvfs "
                  "--exclude-env=ad_dc_no_nss "
                  "--exclude-env=fl2003dc "
                  "--exclude-env=fl2008r2dc "
@@ -123,7 +128,6 @@ tasks = {
                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                 ("clean", "make clean", "text/plain")],
 
-    # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                     ("make", "make -j", "text/plain"),
@@ -134,7 +138,6 @@ tasks = {
                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                     ("clean", "make clean", "text/plain")],
 
-    # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                            ("make", "make -j", "text/plain"),
@@ -143,7 +146,6 @@ tasks = {
                             "--include-env=fileserver'", "text/plain"),
                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 
-    # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                       ("make", "make -j", "text/plain"),
@@ -157,7 +159,6 @@ tasks = {
                        "--include-env=ad_member_idmap_ad'", "text/plain"),
                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 
-    # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                         ("make", "make -j", "text/plain"),
@@ -171,6 +172,19 @@ tasks = {
                          "text/plain"),
                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 
+    # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
+    # This is currently the longest task, so we don't randomly delay it.
+    "samba-ad-dc-ntvfs": [
+                      ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
+                      ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
+                      ("make", "make -j", "text/plain"),
+                      ("test", "make test FAIL_IMMEDIATELY=1 "
+                       "TESTS='${PY3_ONLY}"
+                       "--include-env=ad_dc_ntvfs "
+                       "'",
+                       "text/plain"),
+                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
+
     # run the backup/restore testenvs separately as they're fairly standalone
     # (and CI seems to max out at ~8 different DCs running at once)
     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),