ci: Run multiple fpgas per job

This consolidates the different FPGA runs into a single job.

Signed-off-by: Steven Koo <steven.koo@ni.com>

2 files changed, 54 insertions, 21 deletions

diff --git a/.ci/utils/format_devtest_junitxml.py b/.ci/utils/format_devtest_junitxml.py
index b941a64e6..39b4d13ed 100644
--- a/.ci/utils/format_devtest_junitxml.py
+++ b/.ci/utils/format_devtest_junitxml.py
@@ -8,15 +8,32 @@ class ClassNameTestCase(TestCase):
 parser = argparse.ArgumentParser()
 parser.add_argument("search_path")
 parser.add_argument("output_name")
+parser.add_argument("--fpgas")
 args = parser.parse_args()
 
-xml = JUnitXml()
-for file in glob.glob(args.search_path + "/**/*.xml", recursive=True):
-    xml += JUnitXml.fromfile(file)
+if args.fpgas:
+    return_xml = JUnitXml()
+    for fpga in args.fpgas.split(','):
+        xml = JUnitXml()
+        for file in glob.glob(args.search_path + "/" + fpga + "/**/*.xml", recursive=True):
+            xml += JUnitXml.fromfile(file)
+        for suite in xml:
+            for case in suite:
+                classname_case = ClassNameTestCase.fromelem(case)
+                if classname_case.name == 'test_all':
+                        classname_case.name = fpga + " " + classname_case.classname
+        return_xml += xml
+    return_xml.write(args.output_name)
 
-for suite in xml:
-    for case in suite:
-        classname_case = ClassNameTestCase.fromelem(case)
-        if classname_case.name == 'test_all':
-            classname_case.name = classname_case.classname
-xml.write(args.output_name)
+else:
+    return_xml = JUnitXml()
+    for file in glob.glob(args.search_path + "/**/*.xml", recursive=True):
+        return_xml += JUnitXml.fromfile(file)
+
+    for suite in return_xml:
+        for case in suite:
+            classname_case = ClassNameTestCase.fromelem(case)
+            if classname_case.name == 'test_all':
+                    classname_case.name = classname_case.classname
+
+    return_xml.write(args.output_name)
diff --git a/.ci/utils/mutex_hardware.py b/.ci/utils/mutex_hardware.py
index 5e16c86fb..4c05bf50b 100644
--- a/.ci/utils/mutex_hardware.py
+++ b/.ci/utils/mutex_hardware.py
@@ -15,12 +15,12 @@ from fabric import Connection
 from pottery import Redlock
 from redis import Redis
 
+bitfile_name = "usrp_{}_fpga_{}.bit"
 
-def jtag_x3xx(jtag_args, redis_server):
+def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server):
     remote_working_dir = "pipeline_fpga"
     vivado_program_jtag = "/opt/Xilinx/Vivado_Lab/2020.1/bin/vivado_lab -mode batch -source {}/viv_hardware_utils.tcl -nolog -nojournal -tclargs program".format(
         remote_working_dir)
-    jtag_server, jtag_serial, fpga_path = jtag_args.split(",")
     print("Waiting on jtag mutex for {}".format(jtag_server), flush=True)
     with Redlock(key="hw_jtag_{}".format(jtag_server),
                  masters=redis_server, auto_release_time=1000 * 60 * 5):
@@ -32,6 +32,7 @@ def jtag_x3xx(jtag_args, redis_server):
                 os.path.join(pathlib.Path(
                     __file__).parent.absolute(), "jtag/viv_hardware_utils.tcl"),
                 remote=remote_working_dir)
+            fpga_path = os.path.join(fpga_folder, bitfile_name.format(dev_model, fpga))
             jtag_host.put(fpga_path, remote=remote_working_dir)
             jtag_host.run(vivado_program_jtag + " " +
                           os.path.join(remote_working_dir, os.path.basename(fpga_path)) +
@@ -39,21 +40,34 @@ def jtag_x3xx(jtag_args, redis_server):
         print("Waiting 15 seconds for device to come back up and for Vivado to close", flush=True)
         time.sleep(15)
 
-
 def main(args):
     redis_server = {Redis.from_url(
         "redis://{}:6379/0".format(args.redis_server))}
     print("Waiting to acquire mutex for {}".format(args.dut_name), flush=True)
     with Redlock(key=args.dut_name, masters=redis_server, auto_release_time=1000 * 60 * args.dut_timeout):
         print("Got mutex for {}".format(args.dut_name), flush=True)
-        if(args.jtag_x3xx != None):
-            jtag_x3xx(args.jtag_x3xx, redis_server)
-        for command in args.test_commands:
-            result = subprocess.run(shlex.split(command))
-            if(result.returncode != 0):
-                sys.exit(result.returncode)
-        sys.exit(0)
+        if args.fpgas:
+            working_dir = os.getcwd()
+            return_code = 0
+            for fpga in args.fpgas.split(','):
+                os.mkdir(os.path.join(working_dir, fpga))
+                os.chdir(os.path.join(working_dir, fpga))
+
+                if args.jtag_x3xx:
+                    jtag_server, jtag_serial, dev_model, fpga_folder = args.jtag_x3xx.split(',')
+                    jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server)
 
+                for command in args.test_commands:
+                    result = subprocess.run(shlex.split(command))
+                    if(return_code == 0):
+                        return_code = result.returncode
+            sys.exit(return_code)
+        else:
+            for command in args.test_commands:
+                result = subprocess.run(shlex.split(command))
+                if(result.returncode != 0):
+                    sys.exit(result.returncode)
+            sys.exit(0)
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -62,8 +76,10 @@ if __name__ == "__main__":
     # Provide fpga_path as a local path and it will be copied
     # to jtag_server.
     parser.add_argument("--jtag_x3xx", type=str,
-                        help="user@jtag_server,jtag_serial,fpga_path")
-    parser.add_argument("--dut_timeout", type=int, default=30,
+                        help="user@jtag_server,jtag_serial,dev_model,fpga_folder")
+    parser.add_argument("--fpgas", type=str,
+                        help="Comma delimited list of FPGAs to test")
+    parser.add_argument("--dut_timeout", type=int, default=60,
                         help="Dut mutex timeout in minutes")
     parser.add_argument("redis_server", type=str,
                         help="Redis server for mutex")