ci: Add n310 into devtests

This commit adds support for reimaging n3xx devices and running
devtest against a n310. This uses Labgrid to control an sdmux,
which we then reimage the device. In the future, other devices can
use this support to run other tests.

Signed-off-by: Steven Koo <steven.koo@ni.com>

8 files changed, 289 insertions, 83 deletions

diff --git a/.ci/templates/job-uhd-devtest-rhombus.yml b/.ci/templates/job-uhd-devtest-rhombus.yml
index d39da70ff..6fc1baa5c 100644
--- a/.ci/templates/job-uhd-devtest-rhombus.yml
+++ b/.ci/templates/job-uhd-devtest-rhombus.yml
@@ -8,6 +8,9 @@ parameters:
 - name: uhdArtifactSource
   type: string
   default: current
+- name: testDevices
+  type: string
+  default: 'x3xx,b2xx,n3xx'
 
 jobs:
 - template: job-uhd-devtest.yml
@@ -20,77 +23,94 @@ jobs:
     uhdArtifactSource: '${{ parameters.uhdArtifactSource }}'
     redisHost: 'sdr-rhombus'
     dutMatrix:
-      rhombus-x300-UBX-0:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: 'rhombus-x300-UBX-0'
-        devSerial: '30A6019'
-        devBus: 'ip'
-        devAddr: '192.168.40.2'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350A6019'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-UBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-UBX-0'
-        devSerial: '3138EF5'
-        devBus: 'ip'
-        devAddr: '192.168.40.3'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635138E98'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-CBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-CBX-0'
-        devSerial: '30796C2'
-        devBus: 'ip'
-        devAddr: '192.168.40.4'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350796C2'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-WBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-WBX-0'
-        devSerial: '30C5BFF'
-        devBus: 'ip'
-        devAddr: '192.168.40.5'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350C5BFF'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-TWINRX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: rhombus-x310-TWINRX-0
-        devSerial: 'F43D13'
-        devBus: 'ip'
-        devAddr: '192.168.40.6'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635F43D13'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x300-SBX-0:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: rhombus-x300-SBX-0
-        devSerial: '32244AD'
-        devBus: 'ip'
-        devAddr: '192.168.40.7'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516352244AD'
-        jtagServer: 'nitest@sdr-rhombus'
+      ${{ if contains(parameters.testDevices, 'x3xx') }}:
+        rhombus-x300-UBX-0:
+          devType: 'x300'
+          devModel: 'x300'
+          devName: 'rhombus-x300-UBX-0'
+          devSerial: '30A6019'
+          devBus: 'ip'
+          devAddr: '192.168.40.2'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350A6019'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-UBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-UBX-0'
+          devSerial: '3138EF5'
+          devBus: 'ip'
+          devAddr: '192.168.40.3'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '251635138E98'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-CBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-CBX-0'
+          devSerial: '30796C2'
+          devBus: 'ip'
+          devAddr: '192.168.40.4'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350796C2'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-WBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-WBX-0'
+          devSerial: '30C5BFF'
+          devBus: 'ip'
+          devAddr: '192.168.40.5'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350C5BFF'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-TWINRX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: rhombus-x310-TWINRX-0
+          devSerial: 'F43D13'
+          devBus: 'ip'
+          devAddr: '192.168.40.6'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '251635F43D13'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x300-SBX-0:
+          devType: 'x300'
+          devModel: 'x300'
+          devName: rhombus-x300-SBX-0
+          devSerial: '32244AD'
+          devBus: 'ip'
+          devAddr: '192.168.40.7'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516352244AD'
+          jtagServer: 'nitest@sdr-rhombus'
+
+      ${{ if contains(parameters.testDevices, 'b2xx') }}:
+        rhombus-b210-0:
+          devType: 'b200'
+          devModel: 'b210'
+          devName: rhombus-b210-0
+          devSerial: '3218D0E'
+          devBus: 'usb'
+          devtestPattern: 'b2xx'
 
-      rhombus-b210-0:
-        devType: 'b200'
-        devModel: 'b210'
-        devName: rhombus-b210-0
-        devSerial: '3218D0E'
-        devBus: 'usb'
-        devtestPattern: 'b2xx'
+      ${{ if contains(parameters.testDevices, 'n3xx') }}:
+        rhombus-n310-0:
+          devType: 'n3xx'
+          devModel: 'n310'
+          devName: rhombus-n310-0
+          devSerial: '3176DF3'
+          devHostname: 'ni-n3xx-3176df3'
+          devBus: 'ip'
+          devAddr: '192.168.20.5'
+          sfpAddrs: '192.168.10.5,192.168.20.5'
+          devFpga: 'HG,XG'
+          devtestPattern: 'n3x0'
+          devSDImage: gnuradio-image-ni-sulfur-rev11-mender.sdimg.bz2
+          devLabgridConfig: .ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
diff --git a/.ci/templates/job-uhd-devtest.yml b/.ci/templates/job-uhd-devtest.yml
index dffcb0e74..24d1985bf 100644
--- a/.ci/templates/job-uhd-devtest.yml
+++ b/.ci/templates/job-uhd-devtest.yml
@@ -65,6 +65,14 @@ jobs:
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: true
 
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: $(devType)-images
+    # Only sync the bz2 sdimg since the bmap
+    # is incompatible with mender
+    patterns: '**/*.bz2'
+    displayName: Download $(devType)-images artifact
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'))
+
   - script: |
       cd $(Build.BinariesDirectory)/uhddev/build
       mkdir -p fpga_images
@@ -84,6 +92,27 @@ jobs:
       export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
       export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
       python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
+        --sdimage $(devType),$(devModel),$(uhd_artifact_directory)/$(devType)-images/$(devSDImage),${{ parameters.uhdSrcDir }}/$(devLabgridConfig),$(devHostname) \
+        --fpgas $(devFpga) \
+        --sfp_addrs $(sfpAddrs) \
+        ${{ parameters.redisHost }} $(devName) \
+        "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
+        "python3 ${{ parameters.uhdSrcDir }}/host/tests/devtest/run_testsuite.py \
+          --src-dir ${{ parameters.uhdSrcDir }}/host/tests/devtest \
+          --devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
+          --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
+          --python-interp python3 --xml"
+    continueOnError: true
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'), eq(variables.devBus, 'ip'))
+    displayName: Run n3xx devtest on $(devName)
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/devtest
+      cd $(Common.TestResultsDirectory)/devtest
+      export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
+      export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
+      export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
+      python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
         ${{ parameters.redisHost }} $(devName) \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args serial=$(devSerial)" \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_image_loader --args serial=$(devSerial),type=$(devType)" \
@@ -102,7 +131,7 @@ jobs:
       export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
       export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
       python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
-        --jtag_x3xx $(jtagServer),$(jtagSerial),$(devModel),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
+        --jtag_x3xx $(devType),$(devModel),$(jtagServer),$(jtagSerial),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
         --fpgas $(devFpga) \
         ${{ parameters.redisHost }} $(devName) \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 7741e22e6..17017f9e2 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -226,17 +226,31 @@ stages:
           toolset: msbuild
           installer: nsis
 
-- stage: test_uhd_stage
-  displayName: Test UHD
-  dependsOn: build_uhd_stage_linux
+- stage: devtest_uhd_x3xx_b2xx_stage
+  displayName: devtest UHD x3xx b2xx
+  dependsOn:
+  - build_uhd_stage_linux
+  jobs:
+  - template: job-uhd-devtest-rhombus.yml
+    parameters:
+      testOS: ubuntu2004
+      uhdSrcDir: $(Build.SourcesDirectory)
+      testDevices: 'x3xx,b2xx'
+
+- stage: devtest_uhd_n3xx_stage
+  displayName: devtest UHD n3xx
+  dependsOn:
+  - build_uhd_stage_linux
+  - build_uhd_embedded_system_images
   jobs:
   - template: job-uhd-devtest-rhombus.yml
     parameters:
       testOS: ubuntu2004
       uhdSrcDir: $(Build.SourcesDirectory)
+      testDevices: 'n3xx'
 
-- stage: test_uhd_embedded_devices_stage
-  displayName: Test UHD Embedded Devices
+- stage: test_uhd_x4xx_stage
+  displayName: Test UHD x4xx
   dependsOn:
   - build_uhd_stage_linux
   - build_uhd_embedded_system_images
diff --git a/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml b/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml
new file mode 100755
index 000000000..43284c5ef
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml
@@ -0,0 +1,17 @@
+rhombus-n310-0:
+  acquired: null
+  acquired_resources: []
+  aliases: []
+  allowed: []
+  changed: 1654034475.1935894
+  comment: ''
+  created: 1654034136.0077882
+  matches:
+  - cls: '*'
+    exporter: '*'
+    group: rhombus-n310-0-group
+    name: null
+    rename: null
+  reservation: null
+  tags: {}
+ 
diff --git a/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml b/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
new file mode 100644
index 000000000..ba22280da
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
@@ -0,0 +1,16 @@
+targets:
+  main:
+    resources:
+      RemotePlace:
+        name: 'rhombus-n310-0'
+    drivers:
+    - SerialDriver:
+        name: 'linux_serial_driver'
+        bindings:
+          port: 'console-linux'
+    - SerialDriver:
+        name: 'scu_serial_driver'
+        bindings:
+          port: 'console-scu'
+    - USBSDMuxDriver: {}
+    - USBStorageDriver: {}
diff --git a/.ci/templates/tests/rhombus-labgrid/docker-compose.yml b/.ci/templates/tests/rhombus-labgrid/docker-compose.yml
new file mode 100755
index 000000000..22a1eb219
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/docker-compose.yml
@@ -0,0 +1,26 @@
+version: '3.3'
+services:
+  coordinator:
+    image: "labgrid-coordinator"
+    volumes:
+      - "./crossbar:/home/root/crossbar:Z"
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    command: bash -c "cp /home/root/crossbar/places.yaml /opt/crossbar/places.yaml &&
+      crossbar start --config /opt/labgrid/.crossbar/config.yaml"
+  exporter:
+    image: "labgrid-exporter"
+    volumes:
+      - "./exporter-conf:/opt/conf:Z"
+      - "/run/udev:/run/udev:ro"
+      - "/dev:/dev"
+    depends_on:
+      - coordinator
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    stdin_open: true
+    privileged: true
+    command: bash -c "set -e &&
+      /opt/wait-for-it/wait-for-it.sh 127.0.0.1:20408 -- labgrid-exporter /opt/conf/exporter.yaml"
diff --git a/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml b/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml
new file mode 100755
index 000000000..c898b8dda
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml
@@ -0,0 +1,17 @@
+rhombus-n310-0-group:
+  console-scu:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
+      ID_USB_INTERFACE_NUM: '01'
+    speed: 115200
+  console-linux:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
+      ID_USB_INTERFACE_NUM: '00'
+    speed: 115200
+  USBSDMuxDevice:
+    match:
+      ID_SERIAL_SHORT: '000000001506'
+
diff --git a/.ci/utils/mutex_hardware.py b/.ci/utils/mutex_hardware.py
index 4c05bf50b..e6fece61f 100644
--- a/.ci/utils/mutex_hardware.py
+++ b/.ci/utils/mutex_hardware.py
@@ -4,6 +4,7 @@
 # into a state where it can be used for testing.
 
 import argparse
+import labgrid
 import os
 import pathlib
 import shlex
@@ -17,7 +18,7 @@ from redis import Redis
 
 bitfile_name = "usrp_{}_fpga_{}.bit"
 
-def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server):
+def jtag_x3xx(dev_type, dev_model, jtag_server, jtag_serial, fpga_folder, fpga, redis_server):
     remote_working_dir = "pipeline_fpga"
     vivado_program_jtag = "/opt/Xilinx/Vivado_Lab/2020.1/bin/vivado_lab -mode batch -source {}/viv_hardware_utils.tcl -nolog -nojournal -tclargs program".format(
         remote_working_dir)
@@ -40,12 +41,69 @@ def jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_serv
         print("Waiting 15 seconds for device to come back up and for Vivado to close", flush=True)
         time.sleep(15)
 
+def set_sfp_addrs(mgmt_addr, sfp_addrs):
+    with Connection(host=mgmt_addr,user='root',connect_kwargs={"password":"", "timeout":120, "banner_timeout":120, "auth_timeout":120}) as dut:
+        for idx, sfp_addr in enumerate(sfp_addrs):
+            dut.run(f"ip link set sfp{idx} down")
+            dut.run(f"ip addr add {sfp_addr}/24 dev sfp{idx}")
+            dut.run(f"ip link set sfp{idx} up")
+    time.sleep(30)
+
+def flash_sdimage(dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr, sfp_addrs):
+    subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} release --kick"))
+    subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} acquire"))
+    env = labgrid.Environment(labgrid_device_yaml)
+    target = env.get_target()
+
+    print("Powering down DUT", flush=True)
+    cp = target.get_driver(labgrid.protocol.ConsoleProtocol, name="scu_serial_driver")
+    cp.write("\napshutdown\n".encode())
+
+    print("Switching SDMux to Host", flush=True)
+    sdmux = target.get_driver(labgrid.driver.USBSDMuxDriver)
+    sdmux.set_mode('host')
+
+    print(f"Writing SDMux using {sdimage_path}", flush=True)
+    massstore = target.get_driver(labgrid.driver.USBStorageDriver)
+    # This uses bmaptool in --nobmap mode to write the sdimg,
+    # since it automatically decompresses the image. Do not
+    # include the .bmap file since it can cause corruption with mender.
+    # See: https://github.com/mendersoftware/meta-mender/pull/1076
+    massstore.write_image(sdimage_path, mode=labgrid.driver.usbstoragedriver.Mode.BMAPTOOL)
+    time.sleep(30)
+
+    print("Switching SDMux to DUT", flush=True)
+    sdmux.set_mode('dut')
+    time.sleep(30)
+
+    print("Powering on DUT", flush=True)
+    cp.write("\npowerbtn\n".encode())
+
+    print("Waiting 2 minutes for device to boot", flush=True)
+    time.sleep(120)
+    known_hosts_path = os.path.expanduser("~/.ssh/known_hosts")
+    subprocess.run(shlex.split(f"ssh-keygen -f \"{known_hosts_path}\" -R \"{mgmt_addr}\""))
+
+    if sfp_addrs:
+        set_sfp_addrs(mgmt_addr, sfp_addrs)
+
+    subprocess.run(shlex.split(f"labgrid-client -c {labgrid_device_yaml} release"))
+
 def main(args):
     redis_server = {Redis.from_url(
         "redis://{}:6379/0".format(args.redis_server))}
     print("Waiting to acquire mutex for {}".format(args.dut_name), flush=True)
     with Redlock(key=args.dut_name, masters=redis_server, auto_release_time=1000 * 60 * args.dut_timeout):
         print("Got mutex for {}".format(args.dut_name), flush=True)
+
+        if args.sdimage:
+            dev_type, dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr = args.sdimage.split(',')
+            if args.sfp_addrs:
+                sfp_addrs = args.sfp_addrs.split(',')
+            else:
+                sfp_addrs = None
+            flash_sdimage(dev_model, sdimage_path, labgrid_device_yaml, mgmt_addr, sfp_addrs)
+
         if args.fpgas:
             working_dir = os.getcwd()
             return_code = 0
@@ -54,8 +112,13 @@ def main(args):
                 os.chdir(os.path.join(working_dir, fpga))
 
                 if args.jtag_x3xx:
-                    jtag_server, jtag_serial, dev_model, fpga_folder = args.jtag_x3xx.split(',')
-                    jtag_x3xx(jtag_server, jtag_serial, dev_model, fpga_folder, fpga, redis_server)
+                    dev_type, dev_model, jtag_server, jtag_serial, fpga_folder = args.jtag_x3xx.split(',')
+                    jtag_x3xx(dev_type, dev_model, jtag_server, jtag_serial, fpga_folder, fpga, redis_server)
+
+                if dev_type and dev_type in ["n3xx", "e3xx"]:
+                    subprocess.run(shlex.split(f"uhd_image_loader --args=mgmt_addr={mgmt_addr},type={dev_type},fpga={fpga}"))
+                    if sfp_addrs:
+                        set_sfp_addrs(mgmt_addr, sfp_addrs)
 
                 for command in args.test_commands:
                     result = subprocess.run(shlex.split(command))
@@ -76,7 +139,11 @@ if __name__ == "__main__":
     # Provide fpga_path as a local path and it will be copied
     # to jtag_server.
     parser.add_argument("--jtag_x3xx", type=str,
-                        help="user@jtag_server,jtag_serial,dev_model,fpga_folder")
+                        help="dev_type,dev_model,user@jtag_server,jtag_serial,fpga_folder")
+    parser.add_argument("--sdimage", type=str,
+                        help="dev_type,dev_model,sdimg_path,labgrid_device_yaml,mgmt_addr")
+    parser.add_argument("--sfp_addrs", type=str,
+                        help="sfp0ip,sfp1ip,...")
     parser.add_argument("--fpgas", type=str,
                         help="Comma delimited list of FPGAs to test")
     parser.add_argument("--dut_timeout", type=int, default=60,