Transition to using static builders.

Added the ability to control if dependencies are installed
for a builder. This allows for AMIs with dependencies preinstalled
to be used. This should speed up slave bootstrapping and provide
more consistent build results.

Author: dinatale2

master/lustrefactory.py

@@ -21,6 +21,9 @@ def do_step_build(step, name):
 def do_step_zfs(step):
     return do_step_build(step, 'buildzfs')
 
+def do_step_installdeps(step):
+    return do_step_build(step, 'installdeps')
+
 @util.renderer
 def dependencyCommand(props):
     args = ["runurl"]
@@ -83,6 +86,8 @@ def getBuildFactory(gerrit_repo, **kwargs):
         command=dependencyCommand,
         decodeRC={0 : SUCCESS, 1 : FAILURE, 2 : WARNINGS, 3 : SKIPPED },
         haltOnFailure=True, logEnviron=False,
+        doStepIf=do_step_installdeps,
+        hideStepIf=lambda results, s: results==SKIPPED,
         description=["installing dependencies"],
         descriptionDone=["installed dependencies"]))
 

master/master.cfg

@@ -73,6 +73,7 @@ def merge_dicts(*args):
 # of scripts to execute and which spl and zfs tags to boot strap with.
 global_props = {
     "bburl"       :      bb_url,
+    "installdeps" :      "no",
     "spltag"      :      "spl-0.6.5.4",
     "zfstag"      :      "zfs-0.6.5.4",
 }
@@ -123,35 +124,35 @@ numSlaves = 3  # number of slaves per builder
 CentOS_6_7_slaves = [
     LustreEC2Slave(
         name="CentOS-6.7-x86_64-buildslave%s" % (str(i+1)),
-        ami="ami-4d5d577d"
+        ami="ami-3a05f55a"
     ) for i in range(0, numSlaves)
 ]
 
 CentOS_7_2_slaves = [
     LustreEC2Slave(
         name="CentOS-7.2-x86_64-buildslave%s" % (str(i+1)),
-        ami="ami-666a7507"
+        ami="ami-d945b5b9"
     ) for i in range(0, numSlaves)
 ]
 
 SLES_12_1_slaves = [
     LustreEC2SuseSlave(
         name="SUSE-12.1-x86_64-buildslave%s" % (str(i+1)),
-        ami="ami-16f61b76"
+        ami="ami-6b77870b"
     ) for i in range(0, numSlaves)
 ]
 
 Ubuntu_12_04_slaves = [
     LustreEC2Slave(
         name="Ubuntu-12.04-x86_64-buildslave%s" % (str(i+1)),
-        ami="ami-155a6125"
+        ami="ami-55788835"
     ) for i in range(0, numSlaves)
 ]
 
 Ubuntu_14_04_slaves = [
     LustreEC2Slave(
         name="Ubuntu-14.04-x86_64-buildslave%s" % (str(i+1)),
-        ami="ami-5189a661"
+        ami="ami-1b76867b"
     ) for i in range(0, numSlaves)
 ]
 

scripts/bb-dependencies.sh

@@ -21,7 +21,7 @@ Amazon*)
 
     # Required utilties.
     sudo yum -y install git rpm-build wget curl lsscsi parted attr dbench \
-        watchdog createrepo python python-pip python-docutils
+        watchdog createrepo python python-pip python-docutils xfig transfig
     ;;
 
 CentOS*)
@@ -44,7 +44,7 @@ CentOS*)
 
     # Required utilties.
     sudo yum -y install git rpm-build wget curl lsscsi parted attr dbench bc \
-        watchdog createrepo mock python python-docutils mdadm
+        watchdog createrepo mock python python-docutils mdadm xfig transfig
 
     # add user to the mock group
     sudo usermod -a -G mock buildbot
@@ -67,7 +67,7 @@ Debian*)
     # Required utilties.
     sudo apt-get --yes install git alien fakeroot wget curl bc \
         lsscsi parted gdebi attr dbench watchdog createrepo \
-        python python-pip python-docutils
+        python python-pip python-docutils xfig transfig
     ;;
 
 Fedora*)
@@ -84,7 +84,7 @@ Fedora*)
 
     # Required utilties.
     sudo dnf -y install git rpm-build wget curl lsscsi parted attr dbench \
-        watchdog createrepo mock python python-pip python-docutils
+        watchdog createrepo mock python python-pip python-docutils xfig transfig
 
     # add user to the mock group
     sudo usermod -a -G mock buildbot
@@ -115,7 +115,7 @@ RHEL*)
     # Required utilties.
     sudo yum -y $EXTRA_REPO install git rpm-build wget curl lsscsi \
         parted attr dbench bc watchdog createrepo mock python \
-        python-pip python-docutils mdadm
+        python-pip python-docutils mdadm xfig transfig
 
     # add user to the mock group
     sudo usermod -a -G mock buildbot
@@ -130,7 +130,7 @@ SUSE*)
 
     # Required utilties.
     sudo zypper --non-interactive install git rpm-build wget curl \
-        lsscsi parted attr createrepo python python-pip python-docutils
+        lsscsi parted attr createrepo python python-pip python-docutils xfig transfig
 
     # Required development packages.
     sudo zypper --non-interactive --plus-content debug install \
@@ -148,7 +148,7 @@ OpenSUSE*)
 
     # Required utilties.
     sudo zypper --non-interactive install git rpm-build wget curl \
-        lsscsi parted attr createrepo python python-pip python-docutils
+        lsscsi parted attr createrepo python python-pip python-docutils xfig transfig
 
     # Required development packages.
     sudo zypper --non-interactive --plus-content debug install \
@@ -165,7 +165,7 @@ Ubuntu*)
     # Required utilties.
     sudo apt-get --yes install git alien fakeroot wget curl \
         lsscsi parted gdebi attr dbench watchdog \
-        python python-pip python-docutils
+        python python-pip python-docutils xfig transfig
 
     # Required development libraries
     sudo apt-get --yes install linux-headers-$(uname -r) \