From fe2e24e79e3e5d714e2bf30da89ee5cd71dcb399 Mon Sep 17 00:00:00 2001
From: Virendra Kakade <virendra.kakade@ni.com>
Date: Mon, 28 Mar 2022 14:32:05 -0500
Subject: tests: enable automated streaming tests on X410
Enable automated streaming tests to run on X410.
Disable tests on non-X410 devices for now.
Run only DPDK tests since the setup can not withstand
4Rx 4Tx and 4FDx in non-DPDK mode in manual testing.
And we need to enable running these tests for 100GbE testing.
Adjust streaming test thresholds to new values which seemed to work fine
in manual testing. Might make them tighter in future based on more data.
---
.ci/templates/job-uhd-streaming-tests-beauty.yml | 14 ++++++++---
.ci/templates/job-uhd-streaming-tests.yml | 32 +++++++++++++-----------
.ci/templates/stages-uhd-pipeline.yml | 2 +-
.ci/uhd-pipeline-pr.yml | 2 +-
.ci/uhd-pipeline.yml | 2 +-
host/tests/pytests/test_streaming.py | 17 +++++++------
6 files changed, 39 insertions(+), 30 deletions(-)
diff --git a/.ci/templates/job-uhd-streaming-tests-beauty.yml b/.ci/templates/job-uhd-streaming-tests-beauty.yml
index 0f9065830..f58101590 100644
--- a/.ci/templates/job-uhd-streaming-tests-beauty.yml
+++ b/.ci/templates/job-uhd-streaming-tests-beauty.yml
@@ -17,9 +17,15 @@ jobs:
toolset: 'make'
uhdSrcDir: '${{ parameters.uhdSrcDir }}'
dutMatrix:
- beauty-N320-0 XG:
- dutName: 'beauty-N320-0'
- dutType: 'N320'
+ # beauty-N320-0 XG:
+ # dutName: 'beauty-N320-0'
+ # dutType: 'N320'
+ # dutAddr: '192.168.10.2'
+ # dutSecondAddr: '192.168.20.2'
+ # dutMgmtAddr: '10.0.57.13'
+ beauty-X410-0 CG_400:
+ dutName: 'beauty-X410-0'
+ dutType: 'X410'
dutAddr: '192.168.10.2'
dutSecondAddr: '192.168.20.2'
- dutMgmtAddr: '10.0.57.13'
+ dutMgmtAddr: '10.0.57.29'
diff --git a/.ci/templates/job-uhd-streaming-tests.yml b/.ci/templates/job-uhd-streaming-tests.yml
index 35eb7de54..f36397b6a 100644
--- a/.ci/templates/job-uhd-streaming-tests.yml
+++ b/.ci/templates/job-uhd-streaming-tests.yml
@@ -43,20 +43,20 @@ jobs:
archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
destinationFolder: $(Build.BinariesDirectory)
cleanDestinationFolder: true
- - script: |
- cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
- sudo ./setup.sh --auto
- sleep 5
- displayName: setup interfaces for use without DPDK
- - script: |
- set -x
- export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
- cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
- python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
- --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
- --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
- continueOnError: true
- displayName: Run streaming tests on $(dutName)
+ # - script: |
+ # cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
+ # sudo ./setup.sh --auto
+ # sleep 5
+ # displayName: setup interfaces for use without DPDK
+ # - script: |
+ # set -x
+ # export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
+ # cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
+ # sudo python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
+ # --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
+ # --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
+ # continueOnError: true
+ # displayName: Run streaming tests on $(dutName)
- script: |
cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
sudo ./setup.sh --auto --dpdk
@@ -66,7 +66,9 @@ jobs:
set -x
export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
- sudo --preserve-env=PYTHONPATH python3 -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
+ # Disable creation of __pycache__ files using -B to avoid errors on clean up during next run
+ # as using sudo below creates them with root priveleges.
+ sudo --preserve-env=PYTHONPATH python3 -B -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
--addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
--uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName)-dpdk.xml
continueOnError: true
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 66a07d43d..b2386b351 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -27,7 +27,7 @@ parameters:
type: string
- name: run_streaming_tests
type: boolean
- default: false
+ default: true
- name: build_sdk
type: boolean
default: false
diff --git a/.ci/uhd-pipeline-pr.yml b/.ci/uhd-pipeline-pr.yml
index d847bc33f..d3a8077e0 100644
--- a/.ci/uhd-pipeline-pr.yml
+++ b/.ci/uhd-pipeline-pr.yml
@@ -40,7 +40,7 @@ parameters:
- name: run_streaming_tests
type: boolean
displayName: Run Streaming Tests
- default: False
+ default: true
- name: build_sdk
type: boolean
displayName: Build embedded image SDKs
diff --git a/.ci/uhd-pipeline.yml b/.ci/uhd-pipeline.yml
index cc0308d3e..3079a2c9e 100644
--- a/.ci/uhd-pipeline.yml
+++ b/.ci/uhd-pipeline.yml
@@ -32,7 +32,7 @@ parameters:
- name: run_streaming_tests
type: boolean
displayName: Run Streaming Tests
- default: False
+ default: true
- name: build_sdk
type: boolean
displayName: Build embedded image SDKs
diff --git a/host/tests/pytests/test_streaming.py b/host/tests/pytests/test_streaming.py
index 9d3f566fc..741c51a07 100644
--- a/host/tests/pytests/test_streaming.py
+++ b/host/tests/pytests/test_streaming.py
@@ -263,16 +263,17 @@ def test_streaming(pytestconfig, dut_type, use_dpdk, dual_SFP, rate, rx_rate, rx
print(batch_run_benchmark_rate.get_summary_string(stats, iterations, benchmark_rate_params))
# compare results against thresholds
- dropped_samps_threshold = 0
- overruns_threshold = 2
- rx_timeouts_threshold = 0
- rx_seq_err_threshold = 0
+ # TODO: Have non adhoc better thresholds.
+ dropped_samps_threshold = 50
+ overruns_threshold = 50
+ rx_timeouts_threshold = 50
+ rx_seq_err_threshold = 50
- underruns_threshold = 2
- tx_timeouts_threshold = 0
- tx_seq_err_threshold = 0
+ underruns_threshold = 50
+ tx_timeouts_threshold = 50
+ tx_seq_err_threshold = 50
- late_cmds_threshold = 0
+ late_cmds_threshold = 50
# TODO: define custom failed assertion explanations to avoid extra output
# https://docs.pytest.org/en/6.2.x/assert.html#defining-your-own-explanation-for-failed-assertions
--
cgit v1.2.3