ci: Add n310 into devtests

This commit adds support for reimaging n3xx devices and running
devtest against a n310. This uses Labgrid to control an sdmux,
which we then reimage the device. In the future, other devices can
use this support to run other tests.

Signed-off-by: Steven Koo <steven.koo@ni.com>

7 files changed, 218 insertions, 79 deletions

diff --git a/.ci/templates/job-uhd-devtest-rhombus.yml b/.ci/templates/job-uhd-devtest-rhombus.yml
index d39da70ff..6fc1baa5c 100644
--- a/.ci/templates/job-uhd-devtest-rhombus.yml
+++ b/.ci/templates/job-uhd-devtest-rhombus.yml
@@ -8,6 +8,9 @@ parameters:
 - name: uhdArtifactSource
   type: string
   default: current
+- name: testDevices
+  type: string
+  default: 'x3xx,b2xx,n3xx'
 
 jobs:
 - template: job-uhd-devtest.yml
@@ -20,77 +23,94 @@ jobs:
     uhdArtifactSource: '${{ parameters.uhdArtifactSource }}'
     redisHost: 'sdr-rhombus'
     dutMatrix:
-      rhombus-x300-UBX-0:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: 'rhombus-x300-UBX-0'
-        devSerial: '30A6019'
-        devBus: 'ip'
-        devAddr: '192.168.40.2'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350A6019'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-UBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-UBX-0'
-        devSerial: '3138EF5'
-        devBus: 'ip'
-        devAddr: '192.168.40.3'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635138E98'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-CBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-CBX-0'
-        devSerial: '30796C2'
-        devBus: 'ip'
-        devAddr: '192.168.40.4'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350796C2'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-WBX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: 'rhombus-x310-WBX-0'
-        devSerial: '30C5BFF'
-        devBus: 'ip'
-        devAddr: '192.168.40.5'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516350C5BFF'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x310-TWINRX-0:
-        devType: 'x300'
-        devModel: 'x310'
-        devName: rhombus-x310-TWINRX-0
-        devSerial: 'F43D13'
-        devBus: 'ip'
-        devAddr: '192.168.40.6'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '251635F43D13'
-        jtagServer: 'nitest@sdr-rhombus'
-      rhombus-x300-SBX-0:
-        devType: 'x300'
-        devModel: 'x300'
-        devName: rhombus-x300-SBX-0
-        devSerial: '32244AD'
-        devBus: 'ip'
-        devAddr: '192.168.40.7'
-        devFpga: 'HG,XG'
-        devtestPattern: 'x3x0'
-        jtagSerial: '2516352244AD'
-        jtagServer: 'nitest@sdr-rhombus'
+      ${{ if contains(parameters.testDevices, 'x3xx') }}:
+        rhombus-x300-UBX-0:
+          devType: 'x300'
+          devModel: 'x300'
+          devName: 'rhombus-x300-UBX-0'
+          devSerial: '30A6019'
+          devBus: 'ip'
+          devAddr: '192.168.40.2'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350A6019'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-UBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-UBX-0'
+          devSerial: '3138EF5'
+          devBus: 'ip'
+          devAddr: '192.168.40.3'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '251635138E98'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-CBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-CBX-0'
+          devSerial: '30796C2'
+          devBus: 'ip'
+          devAddr: '192.168.40.4'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350796C2'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-WBX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: 'rhombus-x310-WBX-0'
+          devSerial: '30C5BFF'
+          devBus: 'ip'
+          devAddr: '192.168.40.5'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516350C5BFF'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x310-TWINRX-0:
+          devType: 'x300'
+          devModel: 'x310'
+          devName: rhombus-x310-TWINRX-0
+          devSerial: 'F43D13'
+          devBus: 'ip'
+          devAddr: '192.168.40.6'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '251635F43D13'
+          jtagServer: 'nitest@sdr-rhombus'
+        rhombus-x300-SBX-0:
+          devType: 'x300'
+          devModel: 'x300'
+          devName: rhombus-x300-SBX-0
+          devSerial: '32244AD'
+          devBus: 'ip'
+          devAddr: '192.168.40.7'
+          devFpga: 'HG,XG'
+          devtestPattern: 'x3x0'
+          jtagSerial: '2516352244AD'
+          jtagServer: 'nitest@sdr-rhombus'
+
+      ${{ if contains(parameters.testDevices, 'b2xx') }}:
+        rhombus-b210-0:
+          devType: 'b200'
+          devModel: 'b210'
+          devName: rhombus-b210-0
+          devSerial: '3218D0E'
+          devBus: 'usb'
+          devtestPattern: 'b2xx'
 
-      rhombus-b210-0:
-        devType: 'b200'
-        devModel: 'b210'
-        devName: rhombus-b210-0
-        devSerial: '3218D0E'
-        devBus: 'usb'
-        devtestPattern: 'b2xx'
+      ${{ if contains(parameters.testDevices, 'n3xx') }}:
+        rhombus-n310-0:
+          devType: 'n3xx'
+          devModel: 'n310'
+          devName: rhombus-n310-0
+          devSerial: '3176DF3'
+          devHostname: 'ni-n3xx-3176df3'
+          devBus: 'ip'
+          devAddr: '192.168.20.5'
+          sfpAddrs: '192.168.10.5,192.168.20.5'
+          devFpga: 'HG,XG'
+          devtestPattern: 'n3x0'
+          devSDImage: gnuradio-image-ni-sulfur-rev11-mender.sdimg.bz2
+          devLabgridConfig: .ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
diff --git a/.ci/templates/job-uhd-devtest.yml b/.ci/templates/job-uhd-devtest.yml
index dffcb0e74..24d1985bf 100644
--- a/.ci/templates/job-uhd-devtest.yml
+++ b/.ci/templates/job-uhd-devtest.yml
@@ -65,6 +65,14 @@ jobs:
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: true
 
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: $(devType)-images
+    # Only sync the bz2 sdimg since the bmap
+    # is incompatible with mender
+    patterns: '**/*.bz2'
+    displayName: Download $(devType)-images artifact
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'))
+
   - script: |
       cd $(Build.BinariesDirectory)/uhddev/build
       mkdir -p fpga_images
@@ -84,6 +92,27 @@ jobs:
       export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
       export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
       python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
+        --sdimage $(devType),$(devModel),$(uhd_artifact_directory)/$(devType)-images/$(devSDImage),${{ parameters.uhdSrcDir }}/$(devLabgridConfig),$(devHostname) \
+        --fpgas $(devFpga) \
+        --sfp_addrs $(sfpAddrs) \
+        ${{ parameters.redisHost }} $(devName) \
+        "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
+        "python3 ${{ parameters.uhdSrcDir }}/host/tests/devtest/run_testsuite.py \
+          --src-dir ${{ parameters.uhdSrcDir }}/host/tests/devtest \
+          --devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
+          --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
+          --python-interp python3 --xml"
+    continueOnError: true
+    condition: and(succeeded(), eq(variables.devType, 'n3xx'), eq(variables.devBus, 'ip'))
+    displayName: Run n3xx devtest on $(devName)
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/devtest
+      cd $(Common.TestResultsDirectory)/devtest
+      export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
+      export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
+      export UHD_IMAGES_DIR=$(Build.BinariesDirectory)/uhddev/build/fpga_images
+      python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
         ${{ parameters.redisHost }} $(devName) \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args serial=$(devSerial)" \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_image_loader --args serial=$(devSerial),type=$(devType)" \
@@ -102,7 +131,7 @@ jobs:
       export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
       export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
       python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
-        --jtag_x3xx $(jtagServer),$(jtagSerial),$(devModel),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
+        --jtag_x3xx $(devType),$(devModel),$(jtagServer),$(jtagSerial),$(Build.BinariesDirectory)/uhddev/build/fpga_images/ \
         --fpgas $(devFpga) \
         ${{ parameters.redisHost }} $(devName) \
         "$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 7741e22e6..17017f9e2 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -226,17 +226,31 @@ stages:
           toolset: msbuild
           installer: nsis
 
-- stage: test_uhd_stage
-  displayName: Test UHD
-  dependsOn: build_uhd_stage_linux
+- stage: devtest_uhd_x3xx_b2xx_stage
+  displayName: devtest UHD x3xx b2xx
+  dependsOn:
+  - build_uhd_stage_linux
+  jobs:
+  - template: job-uhd-devtest-rhombus.yml
+    parameters:
+      testOS: ubuntu2004
+      uhdSrcDir: $(Build.SourcesDirectory)
+      testDevices: 'x3xx,b2xx'
+
+- stage: devtest_uhd_n3xx_stage
+  displayName: devtest UHD n3xx
+  dependsOn:
+  - build_uhd_stage_linux
+  - build_uhd_embedded_system_images
   jobs:
   - template: job-uhd-devtest-rhombus.yml
     parameters:
       testOS: ubuntu2004
       uhdSrcDir: $(Build.SourcesDirectory)
+      testDevices: 'n3xx'
 
-- stage: test_uhd_embedded_devices_stage
-  displayName: Test UHD Embedded Devices
+- stage: test_uhd_x4xx_stage
+  displayName: Test UHD x4xx
   dependsOn:
   - build_uhd_stage_linux
   - build_uhd_embedded_system_images
diff --git a/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml b/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml
new file mode 100755
index 000000000..43284c5ef
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/crossbar/places.yaml
@@ -0,0 +1,17 @@
+rhombus-n310-0:
+  acquired: null
+  acquired_resources: []
+  aliases: []
+  allowed: []
+  changed: 1654034475.1935894
+  comment: ''
+  created: 1654034136.0077882
+  matches:
+  - cls: '*'
+    exporter: '*'
+    group: rhombus-n310-0-group
+    name: null
+    rename: null
+  reservation: null
+  tags: {}
+ 
diff --git a/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml b/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
new file mode 100644
index 000000000..ba22280da
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/device-configs/rhombus-n310-0.yml
@@ -0,0 +1,16 @@
+targets:
+  main:
+    resources:
+      RemotePlace:
+        name: 'rhombus-n310-0'
+    drivers:
+    - SerialDriver:
+        name: 'linux_serial_driver'
+        bindings:
+          port: 'console-linux'
+    - SerialDriver:
+        name: 'scu_serial_driver'
+        bindings:
+          port: 'console-scu'
+    - USBSDMuxDriver: {}
+    - USBStorageDriver: {}
diff --git a/.ci/templates/tests/rhombus-labgrid/docker-compose.yml b/.ci/templates/tests/rhombus-labgrid/docker-compose.yml
new file mode 100755
index 000000000..22a1eb219
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/docker-compose.yml
@@ -0,0 +1,26 @@
+version: '3.3'
+services:
+  coordinator:
+    image: "labgrid-coordinator"
+    volumes:
+      - "./crossbar:/home/root/crossbar:Z"
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    command: bash -c "cp /home/root/crossbar/places.yaml /opt/crossbar/places.yaml &&
+      crossbar start --config /opt/labgrid/.crossbar/config.yaml"
+  exporter:
+    image: "labgrid-exporter"
+    volumes:
+      - "./exporter-conf:/opt/conf:Z"
+      - "/run/udev:/run/udev:ro"
+      - "/dev:/dev"
+    depends_on:
+      - coordinator
+    tty: true
+    network_mode: "host"
+    restart: unless-stopped
+    stdin_open: true
+    privileged: true
+    command: bash -c "set -e &&
+      /opt/wait-for-it/wait-for-it.sh 127.0.0.1:20408 -- labgrid-exporter /opt/conf/exporter.yaml"
diff --git a/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml b/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml
new file mode 100755
index 000000000..c898b8dda
--- /dev/null
+++ b/.ci/templates/tests/rhombus-labgrid/exporter-conf/exporter.yaml
@@ -0,0 +1,17 @@
+rhombus-n310-0-group:
+  console-scu:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
+      ID_USB_INTERFACE_NUM: '01'
+    speed: 115200
+  console-linux:
+    cls: USBSerialPort
+    match:
+      ID_SERIAL: 'Silicon_Labs_CP2105_Dual_USB_to_UART_Bridge_Controller_0097D446'
+      ID_USB_INTERFACE_NUM: '00'
+    speed: 115200
+  USBSDMuxDevice:
+    match:
+      ID_SERIAL_SHORT: '000000001506'
+