ci: Hardware pytests / devtests in AzDO for n310

This commit adds support for running devtests and pytests in
Azure Pipelines. Devices are intentionally matrixed for adding more
in the future. devtest is turned off by default for now, but
can be enabled in the future.

Signed-off-by: Steven Koo <steven.koo@ni.com>

4 files changed, 179 insertions, 9 deletions

diff --git a/.ci/templates/job-uhd-hardware-tests.yml b/.ci/templates/job-uhd-hardware-tests.yml
new file mode 100644
index 000000000..a2ca9755d
--- /dev/null
+++ b/.ci/templates/job-uhd-hardware-tests.yml
@@ -0,0 +1,99 @@
+parameters:
+- name: dutMatrix
+  type: object
+- name: runDevTest
+  type: boolean
+  default: true
+- name: runPyTest
+  type: boolean
+  default: true
+
+jobs:
+- job: build_uhd_run_hardware_test
+  displayName: Build uhd and run hardware test
+  strategy:
+    matrix: ${{ parameters.dutMatrix }}
+
+  pool:
+    name: 'de-dre-lab'
+    demands: uhd_ats -equals $(pipelineAgent)
+
+  steps:
+  - checkout: ettus-rts
+    clean: true
+
+  - checkout: self
+    clean: true
+
+  - template: steps-build-uhd.yml
+    parameters:
+      uhdSrcDir: $(Build.SourcesDirectory)/uhddev
+      uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/devtest
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \
+        --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \
+        --devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
+        --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
+        --python-interp python3 --xml
+    continueOnError: true
+    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), eq(variables['devBus'], 'ip'))
+    displayName: Run devtest for IP
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/devtest
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \
+        --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \
+        --devtest-pattern $(devtestPattern) --args serial=$(devSerial),type=$(devType) \
+        --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
+        --python-interp python3 --xml
+    continueOnError: true
+    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), ne(variables['devBus'], 'ip'))
+    displayName: Run devtest for PCIe / USB
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/pytest
+      cd $(Build.SourcesDirectory)/ettus-rts/config/remote/python_tests
+
+      export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
+      export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
+      export PYTHONPATH=$(Build.BinariesDirectory)/uhddev/build/python/build/lib:$PYTHONPATH
+
+      python3 automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) \
+        --results_path $(Common.TestResultsDirectory)/pytest
+    continueOnError: true
+    condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true'))
+    displayName: Run pytests
+
+  - script: |
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 $(Build.SourcesDirectory)/uhddev/.ci/utils/format_devtest_junitxml.py \
+        $(Common.TestResultsDirectory)/devtest \
+        $(Common.TestResultsDirectory)/devtest/devtestresults.xml
+    continueOnError: true
+    condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'))
+    displayName: Format devtest xml
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml'
+      testRunTitle: $(devName) devtest
+      buildConfiguration: 'Release'
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    condition: and(eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'))
+    displayName: Upload devtest results
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/pytest/**/*.xml'
+      testRunTitle: $(devName) pytest
+      buildConfiguration: 'Release'
+      mergeTestResults: true
+    condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true'))
+    displayName: Upload pytest results
diff --git a/.ci/templates/steps-build-uhd.yml b/.ci/templates/steps-build-uhd.yml
new file mode 100644
index 000000000..22cb60cc9
--- /dev/null
+++ b/.ci/templates/steps-build-uhd.yml
@@ -0,0 +1,21 @@
+parameters:
+- name: uhdSrcDir
+  type: string
+- name: uhdBuildDir
+  type: string
+
+steps:
+- script: |
+    mkdir -p ${{ parameters.uhdBuildDir }}
+    cd ${{ parameters.uhdBuildDir }}
+    cmake ${{ parameters.uhdSrcDir }}/host
+  displayName: cmake make UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    make -j$(nproc)
+  displayName: make UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    ctest --no-compress-output --output-on-failure -T test
+  continueOnError: true
+  displayName: ctest make UHD
diff --git a/.ci/uhd-python-hardware-tests.yml b/.ci/uhd-python-hardware-tests.yml
index 7c55f36c3..33e9f712f 100644
--- a/.ci/uhd-python-hardware-tests.yml
+++ b/.ci/uhd-python-hardware-tests.yml
@@ -4,12 +4,40 @@ trigger: none
 # no PR triggers
 pr: none
 
-pool:
-  name: de-dre-lab
-  demands:
-  - 'uhd_oss_ats'
-
-steps:
-- checkout: none
-- script: printenv
-  displayName: Print Environment
+parameters:
+- name: run_pebbles_n310_0
+  type: boolean
+  default: true
+- name: runDevTest
+  type: boolean
+  default: false
+- name: runPyTest
+  type: boolean
+  default: true
+
+resources:
+  repositories:
+  - repository: ettus-rts
+    type: github
+    endpoint: EttusResearch
+    name: EttusResearch/ettus-rts
+
+jobs:
+- template: templates/job-uhd-hardware-tests.yml
+  parameters:
+    runDevTest: ${{ parameters.runDevTest }}
+    runPyTest: ${{ parameters.runPyTest }}
+    dutMatrix:
+      pebbles-n310-0:
+        devType: 'n3xx'
+        devModel: 'n310'
+        devName: 'pebbles-n310-0'
+        devSerial: '311FE02'
+        devBus: ip
+        devAddr: '192.168.40.17'
+        devMgmtAddr: 'ni-n3xx-311FE02'
+        devtestPattern: 'n3x0'
+        pytestDUT: 'n310-0'
+        pytestAtsConfig: 'uhd_oss_ats'
+        pipelineAgent: pebbles-agent-1
+        deviceEnabled: ${{ parameters.run_pebbles_n310_0 }}
diff --git a/.ci/utils/format_devtest_junitxml.py b/.ci/utils/format_devtest_junitxml.py
new file mode 100644
index 000000000..b941a64e6
--- /dev/null
+++ b/.ci/utils/format_devtest_junitxml.py
@@ -0,0 +1,22 @@
+from junitparser import JUnitXml, Element, Attr, TestCase
+import argparse
+import glob, os
+
+class ClassNameTestCase(TestCase):
+    classname = Attr('classname')
+
+parser = argparse.ArgumentParser()
+parser.add_argument("search_path")
+parser.add_argument("output_name")
+args = parser.parse_args()
+
+xml = JUnitXml()
+for file in glob.glob(args.search_path + "/**/*.xml", recursive=True):
+    xml += JUnitXml.fromfile(file)
+
+for suite in xml:
+    for case in suite:
+        classname_case = ClassNameTestCase.fromelem(case)
+        if classname_case.name == 'test_all':
+            classname_case.name = classname_case.classname
+xml.write(args.output_name)