diff options
| -rw-r--r-- | .ci/templates/job-uhd-streaming-tests-beauty.yml | 14 | ||||
| -rw-r--r-- | .ci/templates/job-uhd-streaming-tests.yml | 32 | ||||
| -rw-r--r-- | .ci/templates/stages-uhd-pipeline.yml | 2 | ||||
| -rw-r--r-- | .ci/uhd-pipeline-pr.yml | 2 | ||||
| -rw-r--r-- | .ci/uhd-pipeline.yml | 2 | ||||
| -rw-r--r-- | host/tests/pytests/test_streaming.py | 17 |
6 files changed, 39 insertions, 30 deletions
diff --git a/.ci/templates/job-uhd-streaming-tests-beauty.yml b/.ci/templates/job-uhd-streaming-tests-beauty.yml index 0f9065830..f58101590 100644 --- a/.ci/templates/job-uhd-streaming-tests-beauty.yml +++ b/.ci/templates/job-uhd-streaming-tests-beauty.yml @@ -17,9 +17,15 @@ jobs: toolset: 'make' uhdSrcDir: '${{ parameters.uhdSrcDir }}' dutMatrix: - beauty-N320-0 XG: - dutName: 'beauty-N320-0' - dutType: 'N320' + # beauty-N320-0 XG: + # dutName: 'beauty-N320-0' + # dutType: 'N320' + # dutAddr: '192.168.10.2' + # dutSecondAddr: '192.168.20.2' + # dutMgmtAddr: '10.0.57.13' + beauty-X410-0 CG_400: + dutName: 'beauty-X410-0' + dutType: 'X410' dutAddr: '192.168.10.2' dutSecondAddr: '192.168.20.2' - dutMgmtAddr: '10.0.57.13' + dutMgmtAddr: '10.0.57.29' diff --git a/.ci/templates/job-uhd-streaming-tests.yml b/.ci/templates/job-uhd-streaming-tests.yml index 35eb7de54..f36397b6a 100644 --- a/.ci/templates/job-uhd-streaming-tests.yml +++ b/.ci/templates/job-uhd-streaming-tests.yml @@ -43,20 +43,20 @@ jobs: archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz destinationFolder: $(Build.BinariesDirectory) cleanDestinationFolder: true - - script: | - cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance - sudo ./setup.sh --auto - sleep 5 - displayName: setup interfaces for use without DPDK - - script: | - set -x - export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance - cd ${{ parameters.uhdSrcDir }}/host/tests/pytests - python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \ - --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \ - --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml - continueOnError: true - displayName: Run streaming tests on $(dutName) + # - script: | + # cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance + # sudo ./setup.sh --auto + # sleep 5 + # displayName: setup interfaces for use without DPDK + # - script: | + # set -x + # export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance + # cd ${{ parameters.uhdSrcDir }}/host/tests/pytests + # sudo python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \ + # --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \ + # --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml + # continueOnError: true + # displayName: Run streaming tests on $(dutName) - script: | cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance sudo ./setup.sh --auto --dpdk @@ -66,7 +66,9 @@ jobs: set -x export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance cd ${{ parameters.uhdSrcDir }}/host/tests/pytests - sudo --preserve-env=PYTHONPATH python3 -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \ + # Disable creation of __pycache__ files using -B to avoid errors on clean up during next run + # as using sudo below creates them with root priveleges. + sudo --preserve-env=PYTHONPATH python3 -B -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \ --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \ --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName)-dpdk.xml continueOnError: true diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml index 66a07d43d..b2386b351 100644 --- a/.ci/templates/stages-uhd-pipeline.yml +++ b/.ci/templates/stages-uhd-pipeline.yml @@ -27,7 +27,7 @@ parameters: type: string - name: run_streaming_tests type: boolean - default: false + default: true - name: build_sdk type: boolean default: false diff --git a/.ci/uhd-pipeline-pr.yml b/.ci/uhd-pipeline-pr.yml index d847bc33f..d3a8077e0 100644 --- a/.ci/uhd-pipeline-pr.yml +++ b/.ci/uhd-pipeline-pr.yml @@ -40,7 +40,7 @@ parameters: - name: run_streaming_tests type: boolean displayName: Run Streaming Tests - default: False + default: true - name: build_sdk type: boolean displayName: Build embedded image SDKs diff --git a/.ci/uhd-pipeline.yml b/.ci/uhd-pipeline.yml index cc0308d3e..3079a2c9e 100644 --- a/.ci/uhd-pipeline.yml +++ b/.ci/uhd-pipeline.yml @@ -32,7 +32,7 @@ parameters: - name: run_streaming_tests type: boolean displayName: Run Streaming Tests - default: False + default: true - name: build_sdk type: boolean displayName: Build embedded image SDKs diff --git a/host/tests/pytests/test_streaming.py b/host/tests/pytests/test_streaming.py index 9d3f566fc..741c51a07 100644 --- a/host/tests/pytests/test_streaming.py +++ b/host/tests/pytests/test_streaming.py @@ -263,16 +263,17 @@ def test_streaming(pytestconfig, dut_type, use_dpdk, dual_SFP, rate, rx_rate, rx print(batch_run_benchmark_rate.get_summary_string(stats, iterations, benchmark_rate_params)) # compare results against thresholds - dropped_samps_threshold = 0 - overruns_threshold = 2 - rx_timeouts_threshold = 0 - rx_seq_err_threshold = 0 + # TODO: Have non adhoc better thresholds. + dropped_samps_threshold = 50 + overruns_threshold = 50 + rx_timeouts_threshold = 50 + rx_seq_err_threshold = 50 - underruns_threshold = 2 - tx_timeouts_threshold = 0 - tx_seq_err_threshold = 0 + underruns_threshold = 50 + tx_timeouts_threshold = 50 + tx_seq_err_threshold = 50 - late_cmds_threshold = 0 + late_cmds_threshold = 50 # TODO: define custom failed assertion explanations to avoid extra output # https://docs.pytest.org/en/6.2.x/assert.html#defining-your-own-explanation-for-failed-assertions |