From a0fcdec7d5f344a5fd7873585a621bef33c25b85 Mon Sep 17 00:00:00 2001 From: Steven Koo Date: Mon, 22 Feb 2021 14:44:18 -0600 Subject: ci: Hardware pytests / devtests in AzDO for n310 This commit adds support for running devtests and pytests in Azure Pipelines. Devices are intentionally matrixed for adding more in the future. devtest is turned off by default for now, but can be enabled in the future. Signed-off-by: Steven Koo --- .ci/templates/job-uhd-hardware-tests.yml | 99 ++++++++++++++++++++++++++++++++ .ci/templates/steps-build-uhd.yml | 21 +++++++ .ci/uhd-python-hardware-tests.yml | 46 ++++++++++++--- .ci/utils/format_devtest_junitxml.py | 22 +++++++ 4 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 .ci/templates/job-uhd-hardware-tests.yml create mode 100644 .ci/templates/steps-build-uhd.yml create mode 100644 .ci/utils/format_devtest_junitxml.py (limited to '.ci') diff --git a/.ci/templates/job-uhd-hardware-tests.yml b/.ci/templates/job-uhd-hardware-tests.yml new file mode 100644 index 000000000..a2ca9755d --- /dev/null +++ b/.ci/templates/job-uhd-hardware-tests.yml @@ -0,0 +1,99 @@ +parameters: +- name: dutMatrix + type: object +- name: runDevTest + type: boolean + default: true +- name: runPyTest + type: boolean + default: true + +jobs: +- job: build_uhd_run_hardware_test + displayName: Build uhd and run hardware test + strategy: + matrix: ${{ parameters.dutMatrix }} + + pool: + name: 'de-dre-lab' + demands: uhd_ats -equals $(pipelineAgent) + + steps: + - checkout: ettus-rts + clean: true + + - checkout: self + clean: true + + - template: steps-build-uhd.yml + parameters: + uhdSrcDir: $(Build.SourcesDirectory)/uhddev + uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build + + - script: | + mkdir -p $(Common.TestResultsDirectory)/devtest + cd $(Common.TestResultsDirectory)/devtest + python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \ + --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \ + --devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \ + --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \ + --python-interp python3 --xml + continueOnError: true + condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), eq(variables['devBus'], 'ip')) + displayName: Run devtest for IP + + - script: | + mkdir -p $(Common.TestResultsDirectory)/devtest + cd $(Common.TestResultsDirectory)/devtest + python3 $(Build.SourcesDirectory)/uhddev/host/tests/devtest/run_testsuite.py \ + --src-dir $(Build.SourcesDirectory)/uhddev/host/tests/devtest \ + --devtest-pattern $(devtestPattern) --args serial=$(devSerial),type=$(devType) \ + --build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \ + --python-interp python3 --xml + continueOnError: true + condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true'), ne(variables['devBus'], 'ip')) + displayName: Run devtest for PCIe / USB + + - script: | + mkdir -p $(Common.TestResultsDirectory)/pytest + cd $(Build.SourcesDirectory)/ettus-rts/config/remote/python_tests + + export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH + export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH + export PYTHONPATH=$(Build.BinariesDirectory)/uhddev/build/python/build/lib:$PYTHONPATH + + python3 automated_main.py --ats_config $(pytestAtsConfig) --dut $(pytestDUT) \ + --results_path $(Common.TestResultsDirectory)/pytest + continueOnError: true + condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true')) + displayName: Run pytests + + - script: | + cd $(Common.TestResultsDirectory)/devtest + python3 $(Build.SourcesDirectory)/uhddev/.ci/utils/format_devtest_junitxml.py \ + $(Common.TestResultsDirectory)/devtest \ + $(Common.TestResultsDirectory)/devtest/devtestresults.xml + continueOnError: true + condition: and(succeeded(), eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true')) + displayName: Format devtest xml + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml' + testRunTitle: $(devName) devtest + buildConfiguration: 'Release' + mergeTestResults: true + failTaskOnFailedTests: true + condition: and(eq('${{ parameters.runDevTest }}', true), eq(variables['deviceEnabled'], 'true')) + displayName: Upload devtest results + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '$(Common.TestResultsDirectory)/pytest/**/*.xml' + testRunTitle: $(devName) pytest + buildConfiguration: 'Release' + mergeTestResults: true + condition: and(succeeded(), eq('${{ parameters.runPyTest }}', true), eq(variables['deviceEnabled'], 'true')) + displayName: Upload pytest results diff --git a/.ci/templates/steps-build-uhd.yml b/.ci/templates/steps-build-uhd.yml new file mode 100644 index 000000000..22cb60cc9 --- /dev/null +++ b/.ci/templates/steps-build-uhd.yml @@ -0,0 +1,21 @@ +parameters: +- name: uhdSrcDir + type: string +- name: uhdBuildDir + type: string + +steps: +- script: | + mkdir -p ${{ parameters.uhdBuildDir }} + cd ${{ parameters.uhdBuildDir }} + cmake ${{ parameters.uhdSrcDir }}/host + displayName: cmake make UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + make -j$(nproc) + displayName: make UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + ctest --no-compress-output --output-on-failure -T test + continueOnError: true + displayName: ctest make UHD diff --git a/.ci/uhd-python-hardware-tests.yml b/.ci/uhd-python-hardware-tests.yml index 7c55f36c3..33e9f712f 100644 --- a/.ci/uhd-python-hardware-tests.yml +++ b/.ci/uhd-python-hardware-tests.yml @@ -4,12 +4,40 @@ trigger: none # no PR triggers pr: none -pool: - name: de-dre-lab - demands: - - 'uhd_oss_ats' - -steps: -- checkout: none -- script: printenv - displayName: Print Environment +parameters: +- name: run_pebbles_n310_0 + type: boolean + default: true +- name: runDevTest + type: boolean + default: false +- name: runPyTest + type: boolean + default: true + +resources: + repositories: + - repository: ettus-rts + type: github + endpoint: EttusResearch + name: EttusResearch/ettus-rts + +jobs: +- template: templates/job-uhd-hardware-tests.yml + parameters: + runDevTest: ${{ parameters.runDevTest }} + runPyTest: ${{ parameters.runPyTest }} + dutMatrix: + pebbles-n310-0: + devType: 'n3xx' + devModel: 'n310' + devName: 'pebbles-n310-0' + devSerial: '311FE02' + devBus: ip + devAddr: '192.168.40.17' + devMgmtAddr: 'ni-n3xx-311FE02' + devtestPattern: 'n3x0' + pytestDUT: 'n310-0' + pytestAtsConfig: 'uhd_oss_ats' + pipelineAgent: pebbles-agent-1 + deviceEnabled: ${{ parameters.run_pebbles_n310_0 }} diff --git a/.ci/utils/format_devtest_junitxml.py b/.ci/utils/format_devtest_junitxml.py new file mode 100644 index 000000000..b941a64e6 --- /dev/null +++ b/.ci/utils/format_devtest_junitxml.py @@ -0,0 +1,22 @@ +from junitparser import JUnitXml, Element, Attr, TestCase +import argparse +import glob, os + +class ClassNameTestCase(TestCase): + classname = Attr('classname') + +parser = argparse.ArgumentParser() +parser.add_argument("search_path") +parser.add_argument("output_name") +args = parser.parse_args() + +xml = JUnitXml() +for file in glob.glob(args.search_path + "/**/*.xml", recursive=True): + xml += JUnitXml.fromfile(file) + +for suite in xml: + for case in suite: + classname_case = ClassNameTestCase.fromelem(case) + if classname_case.name == 'test_all': + classname_case.name = classname_case.classname +xml.write(args.output_name) -- cgit v1.2.3