tests: enable automated streaming tests on X410

Enable automated streaming tests to run on X410.
Disable tests on non-X410 devices for now.

Run only DPDK tests since the setup can not withstand
4Rx 4Tx and 4FDx in non-DPDK mode in manual testing.
And we need to enable running these tests for 100GbE testing.

Adjust streaming test thresholds to new values which seemed to work fine
in manual testing. Might make them tighter in future based on more data.

6 files changed, 39 insertions, 30 deletions

diff --git a/.ci/templates/job-uhd-streaming-tests-beauty.yml b/.ci/templates/job-uhd-streaming-tests-beauty.yml
index 0f9065830..f58101590 100644
--- a/.ci/templates/job-uhd-streaming-tests-beauty.yml
+++ b/.ci/templates/job-uhd-streaming-tests-beauty.yml
@@ -17,9 +17,15 @@ jobs:
     toolset: 'make'
     uhdSrcDir: '${{ parameters.uhdSrcDir }}'
     dutMatrix:
-      beauty-N320-0 XG:
-        dutName: 'beauty-N320-0'
-        dutType: 'N320'
+     # beauty-N320-0 XG:
+     #   dutName: 'beauty-N320-0'
+     #   dutType: 'N320'
+     #   dutAddr: '192.168.10.2'
+     #   dutSecondAddr: '192.168.20.2'
+     #   dutMgmtAddr: '10.0.57.13'
+      beauty-X410-0 CG_400:
+        dutName: 'beauty-X410-0'
+        dutType: 'X410'
         dutAddr: '192.168.10.2'
         dutSecondAddr: '192.168.20.2'
-        dutMgmtAddr: '10.0.57.13'
+        dutMgmtAddr: '10.0.57.29'
diff --git a/.ci/templates/job-uhd-streaming-tests.yml b/.ci/templates/job-uhd-streaming-tests.yml
index 35eb7de54..f36397b6a 100644
--- a/.ci/templates/job-uhd-streaming-tests.yml
+++ b/.ci/templates/job-uhd-streaming-tests.yml
@@ -43,20 +43,20 @@ jobs:
       archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: true
-  - script: |
-      cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
-      sudo ./setup.sh --auto
-      sleep 5
-    displayName: setup interfaces for use without DPDK
-  - script: |
-      set -x
-      export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
-      cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
-      python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
-        --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
-        --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
-    continueOnError: true
-    displayName: Run streaming tests on $(dutName)
+  # - script: |
+  #     cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
+  #     sudo ./setup.sh --auto
+  #     sleep 5
+  #   displayName: setup interfaces for use without DPDK
+  # - script: |
+  #     set -x
+  #     export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
+  #     cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
+  #     sudo python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
+  #       --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
+  #       --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
+  #   continueOnError: true
+  #   displayName: Run streaming tests on $(dutName)
   - script: |
       cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
       sudo ./setup.sh --auto --dpdk
@@ -66,7 +66,9 @@ jobs:
       set -x
       export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
       cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
-      sudo --preserve-env=PYTHONPATH python3 -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
+      # Disable creation of __pycache__ files using -B to avoid errors on clean up during next run
+      # as using sudo below creates them with root priveleges.
+      sudo --preserve-env=PYTHONPATH python3 -B -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
         --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
         --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName)-dpdk.xml
     continueOnError: true
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 66a07d43d..b2386b351 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -27,7 +27,7 @@ parameters:
   type: string
 - name: run_streaming_tests
   type: boolean
-  default: false
+  default: true
 - name: build_sdk
   type: boolean
   default: false
diff --git a/.ci/uhd-pipeline-pr.yml b/.ci/uhd-pipeline-pr.yml
index d847bc33f..d3a8077e0 100644
--- a/.ci/uhd-pipeline-pr.yml
+++ b/.ci/uhd-pipeline-pr.yml
@@ -40,7 +40,7 @@ parameters:
 - name: run_streaming_tests
   type: boolean
   displayName: Run Streaming Tests
-  default: False
+  default: true
 - name: build_sdk
   type: boolean
   displayName: Build embedded image SDKs
diff --git a/.ci/uhd-pipeline.yml b/.ci/uhd-pipeline.yml
index cc0308d3e..3079a2c9e 100644
--- a/.ci/uhd-pipeline.yml
+++ b/.ci/uhd-pipeline.yml
@@ -32,7 +32,7 @@ parameters:
 - name: run_streaming_tests
   type: boolean
   displayName: Run Streaming Tests
-  default: False
+  default: true
 - name: build_sdk
   type: boolean
   displayName: Build embedded image SDKs
diff --git a/host/tests/pytests/test_streaming.py b/host/tests/pytests/test_streaming.py
index 9d3f566fc..741c51a07 100644
--- a/host/tests/pytests/test_streaming.py
+++ b/host/tests/pytests/test_streaming.py
@@ -263,16 +263,17 @@ def test_streaming(pytestconfig, dut_type, use_dpdk, dual_SFP, rate, rx_rate, rx
     print(batch_run_benchmark_rate.get_summary_string(stats, iterations, benchmark_rate_params))
 
     # compare results against thresholds
-    dropped_samps_threshold = 0
-    overruns_threshold = 2
-    rx_timeouts_threshold = 0
-    rx_seq_err_threshold = 0
+    # TODO: Have non adhoc better thresholds.
+    dropped_samps_threshold = 50
+    overruns_threshold = 50
+    rx_timeouts_threshold = 50
+    rx_seq_err_threshold = 50
 
-    underruns_threshold = 2
-    tx_timeouts_threshold = 0
-    tx_seq_err_threshold = 0
+    underruns_threshold = 50
+    tx_timeouts_threshold = 50
+    tx_seq_err_threshold = 50
 
-    late_cmds_threshold = 0
+    late_cmds_threshold = 50
 
     # TODO: define custom failed assertion explanations to avoid extra output
     # https://docs.pytest.org/en/6.2.x/assert.html#defining-your-own-explanation-for-failed-assertions