ci: Add x410 test to mono pipeline

This change brings in x410 testing into the uhd
mono pipeline. This also creates a test development
pipeline that executes on uhd mono pipeline as
an upstream artifact source.

Signed-off-by: Steven Koo <steven.koo@ni.com>

5 files changed, 412 insertions, 0 deletions

diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 05e8f4cf3..66a07d43d 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -77,6 +77,11 @@ resources:
     name: EttusResearch/gr-ettusdev
     endpoint: EttusResearch
     ref: maint-3.8-uhd4.0
+  - repository: ettus-rts
+    type: github
+    endpoint: EttusResearch
+    name: EttusResearch/ettus-rts
+    ref: master
 
 stages:
 - stage: build_uhd_stage_linux
@@ -203,6 +208,20 @@ stages:
       testOS: ubuntu2004
       uhdSrcDir: $(Build.SourcesDirectory)
 
+- stage: test_uhd_embedded_devices_stage
+  displayName: Test UHD Embedded Devices
+  dependsOn:
+  - build_uhd_stage_linux
+  - build_uhd_embedded_system_images
+  - build_gnuradio_stage_linux
+  jobs:
+  - template: tests/job-uhd-x4xx-hardware-tests-sdr-test0.yml
+    parameters:
+      testOS: ubuntu1804
+  - template: tests/job-uhd-x4xx-hardware-tests-pebbles.yml
+    parameters:
+      testOS: ubuntu1804
+
 - stage: test_streaming_stage
   displayName: Test UHD Streaming
   dependsOn: build_uhd_stage_linux
diff --git a/.ci/templates/tests/job-uhd-x4xx-hardware-tests-pebbles.yml b/.ci/templates/tests/job-uhd-x4xx-hardware-tests-pebbles.yml
new file mode 100644
index 000000000..d4624a043
--- /dev/null
+++ b/.ci/templates/tests/job-uhd-x4xx-hardware-tests-pebbles.yml
@@ -0,0 +1,31 @@
+parameters:
+- name: testOS
+  type: string
+  values:
+    - ubuntu1804
+- name: uhdArtifactSource
+  type: string
+  default: current
+
+jobs:
+- template: templates/job-uhd-x4xx-hardware-tests.yml
+  parameters:
+    suiteName: 'ettus_pebbles'
+    testOS: '${{ parameters.testOS }}'
+    toolset: 'make'
+    runPyTest: true
+    runDevTest: false
+    runSystemImageTest: false
+    uhdArtifactSource: ${{ parameters.uhdArtifactSource }}
+    xilinxLocation: /opt/Xilinx/Vivado/2019.1
+    dutMatrix:
+      pebbles-x410-0:
+        dutName: 'pebbles-x410-0'
+        dutType: 'x4xx'
+        devtestPattern: 'x4x0'
+        dutEmbeddedImagesArtifact: 'x4xx-images'
+        uartSerial: '2516351DDCC0'
+        pipelineAgent: pebbles-agent-1
+        pytestAtsConfig: uhd_oss_ats
+        pytestDUT: 'x410'
+        sfp0ip: '192.168.30.62'
diff --git a/.ci/templates/tests/job-uhd-x4xx-hardware-tests-sdr-test0.yml b/.ci/templates/tests/job-uhd-x4xx-hardware-tests-sdr-test0.yml
new file mode 100644
index 000000000..a19675129
--- /dev/null
+++ b/.ci/templates/tests/job-uhd-x4xx-hardware-tests-sdr-test0.yml
@@ -0,0 +1,28 @@
+parameters:
+- name: testOS
+  type: string
+  values:
+    - ubuntu1804
+- name: uhdArtifactSource
+  type: string
+  default: current
+
+jobs:
+- template: templates/job-uhd-x4xx-hardware-tests.yml
+  parameters:
+    suiteName: 'sdr_test0'
+    testOS: '${{ parameters.testOS }}'
+    toolset: 'make'
+    runPyTest: false
+    runDevTest: true
+    runSystemImageTest: true
+    uhdArtifactSource: ${{ parameters.uhdArtifactSource }}
+    xilinxLocation: /opt/Xilinx/SDK/2019.1
+    dutMatrix:
+      sdr-test0-x410-0:
+        dutName: 'sdr-test0-x410-0'
+        dutType: 'x4xx'
+        devtestPattern: 'x4x0'
+        dutEmbeddedImagesArtifact: 'x4xx-images'
+        uartSerial: '2516351FE64E'
+        pipelineAgent: sdr-test0
diff --git a/.ci/templates/tests/templates/job-uhd-x4xx-hardware-tests.yml b/.ci/templates/tests/templates/job-uhd-x4xx-hardware-tests.yml
new file mode 100644
index 000000000..7dcfdf60e
--- /dev/null
+++ b/.ci/templates/tests/templates/job-uhd-x4xx-hardware-tests.yml
@@ -0,0 +1,293 @@
+parameters:
+- name: suiteName
+  type: string
+- name: testOS
+  type: string
+  values:
+    - ubuntu1804
+- name: toolset
+  type: string
+  values:
+    - make
+- name: runPyTest
+  type: boolean
+- name: runSystemImageTest
+  type: boolean
+- name: runDevTest
+  type: boolean
+- name: dutMatrix
+  type: object
+- name: uhdArtifactSource
+  type: string
+  default: current
+- name: xilinxLocation
+  type: string
+
+jobs:
+- job: x4xx_hardware_test_${{ parameters.suiteName }}
+  displayName: x4xx hardware test ${{ parameters.suiteName }}
+  timeoutInMinutes: 120
+  strategy:
+    matrix: ${{ parameters.dutMatrix }}
+
+  pool:
+    name: 'de-dre-lab'
+    demands: uhd_ats -equals $(pipelineAgent)
+
+  steps:
+  - checkout: self
+  - checkout: meta-ettus
+  - checkout: ettus-rts
+
+  - script: |
+      if [ "${{ parameters.uhdArtifactSource }}" = "current" ]; then
+        echo '##vso[task.setvariable variable=uhd_artifact_directory]'$(Agent.BuildDirectory)
+      else
+        echo '##vso[task.setvariable variable=uhd_artifact_directory]'$(Agent.BuildDirectory)/${{ parameters.uhdArtifactSource }}
+      fi
+    displayName: Set uhddev pipeline artifact location
+
+  - script: |
+      rm -rf $(uhd_artifact_directory)/$(dutEmbeddedImagesArtifact)
+      rm -rf $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+      rm -rf $(uhd_artifact_directory)/gnuradio-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Cleanup from prior run
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: $(dutEmbeddedImagesArtifact)
+    displayName: Download $(dutEmbeddedImagesArtifact)
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(uhd_artifact_directory)/$(dutEmbeddedImagesArtifact)/u-boot-jtag-files.zip
+      destinationFolder: $(uhd_artifact_directory)/$(dutEmbeddedImagesArtifact)
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Extract JTAG artifact
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Download uhddev-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      destinationFolder: $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Extract uhddev-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - download: ${{ parameters.uhdArtifactSource }}
+    artifact: gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}
+    displayName: Download gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      destinationFolder: $(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Extract gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }} artifact
+
+  - script: |
+      cd $(Agent.TempDirectory)
+      python3 -m venv venv
+      source venv/bin/activate
+      echo '##vso[task.setvariable variable=USRP_EMB_VENV]'$(realpath venv)
+      echo '##vso[task.setvariable variable=USRP_EMB_IMAGE_PATH]'$(uhd_artifact_directory)/$(dutEmbeddedImagesArtifact)
+      echo '##vso[task.setvariable variable=USRP_EMB_FTDI_SERIAL]'$(uartSerial)
+      python3 -m pip install -U pip
+      pip3 install wheel
+      pip3 install --upgrade $(Build.SourcesDirectory)/meta-ettus-dev/contrib/test/usrp_emb
+    displayName: Create usrp_emb venv
+
+  - script: |
+      source ${{ parameters.xilinxLocation }}/settings64.sh
+      source $USRP_EMB_VENV/bin/activate
+      usrp_emb_x4xx_flash_emmc
+    displayName: Flash eMMC
+
+  - script: |
+      source $USRP_EMB_VENV/bin/activate
+      mkdir -p $(Common.TestResultsDirectory)/sysimg_tests/sysimg_boot_tests
+      cd $(Common.TestResultsDirectory)/sysimg_tests/sysimg_boot_tests
+      usrp_emb_test_x4xx
+    displayName: Run system image boot tests
+    condition: and(succeeded(), ${{ parameters.runSystemImageTest }})
+
+  - script: |
+      source $USRP_EMB_VENV/bin/activate
+      USRP_EMB_TARGET_IP=$(usrp_emb_x4xx_boot_linux)
+      echo Target IP is $USRP_EMB_TARGET_IP
+      echo '##vso[task.setvariable variable=USRP_EMB_TARGET_IP]'$USRP_EMB_TARGET_IP
+    displayName: Boot target, get mgmt IP
+
+  # Setting the timezone on the embedded device is
+  # required to make the test runtimes to look sane.
+  - script: |
+      export HOST_TIME_ZONE=$(cat /etc/timezone)
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "timedatectl set-timezone $HOST_TIME_ZONE"
+    displayName: Set timezone to Host
+
+  - script: |
+      source $USRP_EMB_VENV/bin/activate
+      ssh-keygen -f ~/.ssh/known_hosts -R $USRP_EMB_TARGET_IP
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "ip link set sfp0 down"
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "ip addr add $(sfp0ip)/24 dev sfp0"
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "ip link set sfp0 up"
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "ifconfig sfp0"
+    displayName: Set sfp0 IP
+    condition: and(succeeded(), ${{ parameters.runPyTest }})
+
+  - script: |
+        sleep 60
+    displayName: Wait for device to finish booting
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/sysimg_tests/sysimg_subsystem_tests
+      cd $(Common.TestResultsDirectory)/sysimg_tests/sysimg_subsystem_tests
+      ssh-keygen -f ~/.ssh/known_hosts -R $USRP_EMB_TARGET_IP
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "rm -rf ~/usrp_sysimg_test && mkdir -p ~/usrp_sysimg_test"
+      scp -o StrictHostKeyChecking=no -r $(Build.SourcesDirectory)/meta-ettus-dev/contrib/test/tests root@$USRP_EMB_TARGET_IP:~/usrp_sysimg_test
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "pip3 install unittest-xml-reporting && cd ~/usrp_sysimg_test/tests && python3 -m xmlrunner discover . -v"
+      scp -o StrictHostKeyChecking=no -r root@$USRP_EMB_TARGET_IP:~/usrp_sysimg_test .
+    displayName: Run system image embedded subsystems tests
+    condition: ${{ parameters.runSystemImageTest }}
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/mpm/mpm_unit_tests
+      cd $(Common.TestResultsDirectory)/mpm/mpm_unit_tests
+      ssh-keygen -f ~/.ssh/known_hosts -R $USRP_EMB_TARGET_IP
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "rm -rf ~/mpm_test_run && mkdir -p ~/mpm_test_run"
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "cd ~/mpm_test_run && python3 /usr/lib/usrp_mpm/tests/run_unit_tests.py -x x4xx"
+      scp -o StrictHostKeyChecking=no -r root@$USRP_EMB_TARGET_IP:~/mpm_test_run .
+    displayName: Run MPM unit tests
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/devtest
+      cd $(Common.TestResultsDirectory)/devtest
+      python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \
+        --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \
+        --devtest-pattern $(devtestPattern) --args addr=$USRP_EMB_TARGET_IP,type=$(dutType) \
+        --build-dir $(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev/build \
+        --build-type Release --python-interp python3 --xml
+
+      python3 $(Build.SourcesDirectory)/uhddev/.ci/utils/format_devtest_junitxml.py \
+        $(Common.TestResultsDirectory)/devtest \
+        $(Common.TestResultsDirectory)/devtest/devtestresults.xml
+      echo ""
+    displayName: Run devtest
+    condition: and(succeeded(), ${{ parameters.runDevTest }})
+
+  - script: |
+      cat $(Common.TestResultsDirectory)/devtest/log*.log
+      echo ""
+    displayName: cat devtest log
+    condition: and(succeeded(), ${{ parameters.runDevTest }})
+
+  - script: |
+      export UHD_INSTALL_PATH=$(uhd_artifact_directory)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev/build-installed
+      export GNURADIO_GRETTUS_INSTALL_PATH=$(uhd_artifact_directory)/gnuradio-grettus-${{ parameters.testOS }}-${{ parameters.toolset }}/gnuradio/build-installed
+      export PATH=$UHD_INSTALL_PATH/bin:$PATH
+      export PATH=$GNURADIO_GRETTUS_INSTALL_PATH/bin:$PATH
+      export PYTHON_VERSION=$(python3 -c "import sys; print('python'+str(sys.version_info.major) + '.' + str(sys.version_info.minor))")
+
+      if [ -d "$UHD_INSTALL_PATH/lib64" ]; then
+        export PATH=$UHD_INSTALL_PATH/lib64/uhd/examples:$PATH
+        export LD_LIBRARY_PATH=$UHD_INSTALL_PATH/lib64:$LD_LIBRARY_PATH
+        export LD_LIBRARY_PATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib64:$LD_LIBRARY_PATH
+        export PYTHONPATH=$UHD_INSTALL_PATH/lib64/$PYTHON_VERSION/site-packages:$PYTHONPATH
+        export PYTHONPATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib64/python3/dist-packages:$PYTHONPATH
+      else
+        export PATH=$UHD_INSTALL_PATH/lib/uhd/examples:$PATH
+        export LD_LIBRARY_PATH=$UHD_INSTALL_PATH/lib:$LD_LIBRARY_PATH
+        export LD_LIBRARY_PATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib:$LD_LIBRARY_PATH
+        export PYTHONPATH=$UHD_INSTALL_PATH/lib/$PYTHON_VERSION/site-packages:$PYTHONPATH
+        export PYTHONPATH=$GNURADIO_GRETTUS_INSTALL_PATH/lib/python3/dist-packages:$PYTHONPATH
+      fi
+
+      cd $(Build.SourcesDirectory)/ettus-rts/config/remote/python_tests
+      uhd_find_devices
+      python3 automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) --results_path '$(Common.TestResultsDirectory)/pytest/host'
+      echo ""
+    displayName: Run pytest on host
+    condition: and(succeeded(), ${{ parameters.runPyTest }})
+
+  - script: |
+      mkdir -p $(Common.TestResultsDirectory)/pytest/embedded
+      cd $(Common.TestResultsDirectory)/pytest/embedded
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "pip3 install --upgrade pytest pytest-helpers-namespace pytest-tempdir pytest-salt"
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "rm -rf ~/usrp_pytest && mkdir -p ~/usrp_pytest"
+      scp -o StrictHostKeyChecking=no -r $(Build.SourcesDirectory)/ettus-rts/config/remote/ root@$USRP_EMB_TARGET_IP:~/usrp_pytest
+      ssh -o StrictHostKeyChecking=no -tt root@$USRP_EMB_TARGET_IP "cd ~/usrp_pytest/remote/python_tests && python3 automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) --test_selector pytests/test_gnuradio_sanity.py --run_embedded"
+      scp -o StrictHostKeyChecking=no -r root@$USRP_EMB_TARGET_IP:~/usrp_pytest .
+    displayName: Run pytest on embedded
+    condition: and(succeeded(), ${{ parameters.runPyTest }})
+
+  # Run the mender tests last because they
+  # modify the flashed image on the device
+  - script: |
+      source $USRP_EMB_VENV/bin/activate
+      mkdir -p $(Common.TestResultsDirectory)/sysimg_tests/sysimg_mender_tests
+      cd $(Common.TestResultsDirectory)/sysimg_tests/sysimg_mender_tests
+      source $USRP_EMB_VENV/bin/activate
+      usrp_emb_test_x4xx_mender
+    displayName: Run system image mender tests
+    condition: and(succeeded(), ${{ parameters.runSystemImageTest }})
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/sysimg_tests/**/*.xml'
+      testRunTitle: $(dutName) System Image Tests
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    displayName: Upload $(dutName) System Image Tests
+    condition: ${{ parameters.runSystemImageTest }}
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/mpm/**/*.xml'
+      testRunTitle: $(dutName) mpm unit tests
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    displayName: Upload $(dutName) mpm unit tests
+    condition: always()
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml'
+      testRunTitle: $(dutName) devtests
+      buildConfiguration: 'Release'
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    displayName: Upload $(dutName) devtests
+    condition: ${{ parameters.runDevTest }}
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/pytest/host/**/*.xml'
+      testRunTitle: $(dutName) host pytests
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    displayName: Upload $(dutName) host pytests
+    condition: ${{ parameters.runPyTest }}
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '$(Common.TestResultsDirectory)/pytest/embedded/**/*.xml'
+      testRunTitle: $(dutName) embedded pytests
+      mergeTestResults: true
+      failTaskOnFailedTests: true
+    displayName: Upload $(dutName) embedded pytests
+    condition: ${{ parameters.runPyTest }}
+
+  - publish: $(Common.TestResultsDirectory)
+    artifact: test-logs-x4xx-hardware-$(dutName)-run$(System.JobAttempt)
+    displayName: Upload Test Logs
+    condition: always()
+
diff --git a/.ci/uhd-hardware-test-dev.yml b/.ci/uhd-hardware-test-dev.yml
new file mode 100644
index 000000000..625ecf34a
--- /dev/null
+++ b/.ci/uhd-hardware-test-dev.yml
@@ -0,0 +1,41 @@
+trigger: none
+
+pr: none
+
+resources:
+  pipelines:
+  - pipeline: uhd_mono_pipeline
+    source: 'uhddev mono pipeline'
+    branch: master
+  - pipeline: usrp-kas-pipeline
+    source: 'usrp-kas'
+    branch: zeus-ci
+  repositories:
+  - repository: meta-ettus
+    type: github
+    name: EttusResearch/meta-ettus-dev
+    endpoint: EttusResearch
+    ref: zeus-ci
+  - repository: gr-ettus
+    type: github
+    name: EttusResearch/gr-ettusdev
+    endpoint: EttusResearch
+    ref: maint-3.8-uhd4.0
+  - repository: ettus-rts
+    type: github
+    endpoint: EttusResearch
+    name: EttusResearch/ettus-rts
+    ref: master
+
+stages:
+- stage: test_uhd_embedded_stage
+  displayName: Test Embedded UHD
+  jobs:
+  - template: templates/tests/job-uhd-x4xx-hardware-tests-sdr-test0.yml
+    parameters:
+      testOS: ubuntu1804
+      uhdArtifactSource: uhd_mono_pipeline
+  - template: templates/tests/job-uhd-x4xx-hardware-tests-pebbles.yml
+    parameters:
+      testOS: ubuntu1804
+      uhdArtifactSource: uhd_mono_pipeline