Issues: #1116 - Multi-step fix for RPM and DEB package validation

f5-sdk-dist/Docker/redhat/7/build-rpm.py

@@ -19,11 +19,13 @@
 import glob
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
 import traceback
 
+from collections import namedtuple
 from tempfile import mkdtemp
 
 
@@ -84,6 +86,40 @@ def make_rpms_build(config):
     return rpm_build
 
 
+def change_requires(buildroot, config):
+    spec = buildroot + "/rpmbuild/SPECS/{}.spec".format(config['project'])
+    try:
+        with open(spec, 'r') as fh:
+            contents = fh.read()
+    except IOError as Error:
+        print("Could not open spec file! ({})".format(Error))
+        raise
+    try:
+        with open(spec, 'w') as fh:
+            for line in contents.split("\n"):
+                if 'Requires' in line:
+                    old = line.replace("Requires: ", "")
+                    Req = namedtuple('Req', 'module, modifier, version')
+                    breakout_re = re.compile('([^<=>]+)([<=>]+)([\d]\S+)')
+                    change = 'Requires: '
+                    modifier_format = "{} {} {}, "
+                    for requirement in old.split(' '):
+                        match = breakout_re.search(requirement)
+                        if match:
+                            req = Req(*match.groups())
+                            mod = 'python-' + req.module \
+                                if 'python-' not in req.module and \
+                                'f5-' not in req.module else req.module
+                            change = change + \
+                                modifier_format.format(mod, req.modifier,
+                                                       req.version)
+                    line = change
+                fh.write("{}\n".format(line))
+    except Exception as Error:
+        print("Could not handle change in spec file {}".format(Error))
+        raise
+
+
 def main():
     src_dir = sys.argv[1]
     os.chdir(src_dir)
@@ -103,6 +139,7 @@ def main():
             fh.write('%s_topdir %s/rpmbuild' % ('%', buildroot))
         cmd = "python setup.py bdist_rpm --spec-only --dist-dir rpmbuild/SPECS"
         print(subprocess.check_output([cmd], shell=True))
+        change_requires(buildroot, config)
         cmd = "rpmbuild -ba rpmbuild/SPECS/%s.spec" % project
         print(subprocess.check_output([cmd], shell=True))
         nonarch_pkg = None

f5-sdk-dist/Docker/redhat/install_test/fetch_and_install_deps.py

@@ -106,7 +106,7 @@ class Dependency(object):
 requirements for installation.  It does depend on yum to retrieve subsquent
 dependencies.
     """
-    cmd = "yum install -y %s"
+    install_cmd = "yum install -y %s"
 
     def __init__(self, req):
         match = dep_match_re.search(str(req))
@@ -163,9 +163,9 @@ def install_req(self):
             else self.name
         name = 'python-' + name if 'python-' not in name and \
             '.rpm' not in name else name
-        results, status = runCommand(self.cmd % name)
+        results, status = runCommand(self.install_cmd % name)
         if status:
-            raise InstallError(self.cmd % name, str(self.req),
+            raise InstallError(self.install_cmd % name, str(self.req),
                                msg="Unable to install dep",
                                frame=gfi(cf()), errno=errno.ESPIPE)
 
@@ -181,6 +181,7 @@ class F5Dependency(Dependency):
 actions to perform for its automated installation.
     """
     cmd = "rpm -qRp %s"
+    install_cmd = "rpm -i %s"
 
     def __init__(self, req):
         super(F5Dependency, self).__init__(req)
@@ -253,15 +254,15 @@ def runCommand(cmd):
     """
     output = ""
     try:
-        p = subprocess.Popen(cmd.split(),
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        (output) = p.communicate()[0]
+        output = subprocess.check_output(cmd, shell=True)
     except OSError as e:
         print("Execution failed: [%s:%s] " %
               (cmd, os.listdir('/var/wdir')), str(e))
+    except subprocess.CalledProcessError as Error:
+        print("exceution failed: [{}]".format(Error))
+        return(output, errno.ESPIPE)
     else:
-        return (output, p.returncode)
+        return (output, 0)
     return ('', 99)
 
 
@@ -459,7 +460,9 @@ def fetch_pkg_dependencies(config, pkg_name):
 
     print("Installing Self - %s" % pkg_name)
     try:
-        runCommand('rpm -i %s' % tmp_pkg_name)
+        output, result = runCommand('rpm -i %s 2>&1' % tmp_pkg_name)
+        if not result == 0:
+            raise InstallError("Exit status was {}".format(result))
     except InstallError as error:
         print("Failed to get requirements for %s." % (pkg_name))
         return error
@@ -544,7 +547,14 @@ def main(args):
     if error:
         sys.exit(error.errnum)
     else:
-        sys.exit(0)
+        # last attempt to detect an error:
+        cmd = "rpm -qa | grep {}".format(config['project'])
+        output, status = runCommand(cmd)
+        if status == 0:
+            print("Passed last check:\n{}".format(output))
+            sys.exit(0)
+        print("Failed last level of verification:\n{}".format(cmd))
+        sys.exit(29)
 
 
 if __name__ == '__main__':

f5-sdk-dist/Docker/ubuntu/install_test/fetch_and_install_deps.py

@@ -158,6 +158,9 @@ def install_req(self):
         """
         if 'python-' not in self.name or '(' in self.name:
             return
+        self._install_req()
+
+    def _install_req(self):
         print("Installing %s(v%s)" % (self.name, self.version))
         name = self.pkg_location if hasattr(self, 'pkg_location') \
             else self.name
@@ -233,6 +236,14 @@ def _set_url(self):
         self.pkg_name = pkg_name
         self.url = self.url + pkg_name
 
+    def install_req(self):
+        """install_req
+
+This object method will install the attribute-defined package yielded at
+object creation.  This requires running commands at the command prompt.
+        """
+        self._install_req()
+
 
 def usage():
     """usage
@@ -251,16 +262,12 @@ def runCommand(cmd):
     """
     output = ""
     try:
-        p = subprocess.Popen(cmd.split(),
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        (output) = p.communicate()[0]
-    except OSError as e:
-        print("Execution failed: [%s:%s] " %
-              (cmd, os.listdir('/var/wdir')), str(e))
+        output = subprocess.check_output(cmd, shell=True)
+    except subprocess.CalledProcessError as e:
+        print("Execution failed: [{}]".format(e))
     else:
-        return (output, p.returncode)
-    return ('', 99)
+        return (output, 0)
+    return (str(e), 99)
 
 
 def load_requirements(cfg):
@@ -447,7 +454,10 @@ def fetch_pkg_dependencies(config, pkg_name):
 
     print("Installing Self - %s" % pkg_name)
     try:
-        runCommand('dpkg -i %s' % tmp_pkg_name)
+        output, result = runCommand('dpkg -i %s 2>&1' % tmp_pkg_name)
+        if not result == 0:
+            raise IOError("Result was non-zero! [{}:{}]".format(output,
+                                                                result))
     except InstallError as error:
         print("Failed to get requirements for %s." % (pkg_name))
         return error
@@ -548,8 +558,14 @@ def main(args):
     # Instal from the tmp directory.
     if error:
         sys.exit(error.errnum)
-    else:
+    # last chance to check for failure:
+    cmd = 'dpkg -l | grep {}'.format(config['project'])
+    output, status = runCommand(cmd)
+    if status == 0:
+        print("passed last check:\n{}".format(output))
         sys.exit(0)
+    print("Last check FAILED: {}".format(cmd))
+    sys.exit(99)
 
 
 if __name__ == '__main__':

f5-sdk-dist/deb_dist/stdeb.cfg

@@ -1,5 +1,6 @@
 [DEFAULT]
 Depends:
-    python-f5-icontrol-rest (>=1.3.0), python-f5-icontrol-rest (<2),
-    python-six (>=1.9), python-six (<2) 
-
+     python-six (>=1.9.0), 
+    python-six (<2.0.0), 
+    python-f5-icontrol-rest (>=1.3.0), 
+    python-f5-icontrol-rest (<2.0.0),