From 3b29a89b9f9a8167a595eed2988b89e2a7f2112a Mon Sep 17 00:00:00 2001 From: Steven Koo Date: Tue, 16 Mar 2021 12:39:29 -0500 Subject: ci: Build uhd with Pipelines This commit adds the infrastructure to build uhd on both Linux and Windows using Pipelines. Using the docker images generated by the build docker pipeline, this runs an AzDO container job to build uhd artifacts. It then stores the artifacts to be used in later pipelines or stages. This change supports make and ninja on Linux and msbuild with VS2017/VS2019 x86/x64 on Windows. Signed-off-by: Steven Koo --- .ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile | 9 ++- .ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile | 8 +- .ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile | 9 ++- .ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile | 8 +- .ci/templates/job-get-latest-uhd-docker.yml | 36 +++++++++ .ci/templates/job-uhd-build-src.yml | 99 +++++++++++++++++++++++ .ci/templates/job-uhd-hardware-tests.yml | 2 +- .ci/templates/steps-build-uhd-make.yml | 21 +++++ .ci/templates/steps-build-uhd-msbuild.yml | 33 ++++++++ .ci/templates/steps-build-uhd-ninja.yml | 21 +++++ .ci/templates/steps-build-uhd.yml | 21 ----- .ci/uhd-build-docker-container.yml | 2 + .ci/uhd-pipeline.yml | 40 +++++++++ 13 files changed, 277 insertions(+), 32 deletions(-) create mode 100644 .ci/templates/job-get-latest-uhd-docker.yml create mode 100644 .ci/templates/job-uhd-build-src.yml create mode 100644 .ci/templates/steps-build-uhd-make.yml create mode 100644 .ci/templates/steps-build-uhd-msbuild.yml create mode 100644 .ci/templates/steps-build-uhd-ninja.yml delete mode 100644 .ci/templates/steps-build-uhd.yml create mode 100644 .ci/uhd-pipeline.yml diff --git a/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile b/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile index d608ef4c4..12e198139 100644 --- a/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile +++ b/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile @@ -1,3 +1,5 @@ +# Docker build context must be uhd/uhddev root + # please follow docker best practices # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ @@ -36,7 +38,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \ # This commit uses Boost 1.75 and libusb 1.0.24 git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \ bootstrap-vcpkg.bat + # Add custom UHD vcpkg triplet +COPY host/cmake/vcpkg/* c:/vcpkg/triplets/ RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \ - libusb:x64-windows \ - boost:x64-windows - + libusb:uhd-x64-windows-static-md \ + boost:uhd-x64-windows-static-md diff --git a/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile b/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile index f4cb509fe..6ef77132d 100644 --- a/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile +++ b/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile @@ -1,3 +1,5 @@ +# Docker build context must be uhd/uhddev root + # please follow docker best practices # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ @@ -36,6 +38,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \ # This commit uses Boost 1.75 and libusb 1.0.24 git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \ bootstrap-vcpkg.bat + # Add custom UHD vcpkg triplet +COPY host/cmake/vcpkg/* c:/vcpkg/triplets/ RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \ - libusb:x86-windows \ - boost:x86-windows + libusb:uhd-x86-windows-static-md \ + boost:uhd-x86-windows-static-md diff --git a/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile b/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile index eb8aedcf6..d10b87cac 100644 --- a/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile +++ b/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile @@ -1,3 +1,5 @@ +# Docker build context must be uhd/uhddev root + # please follow docker best practices # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ @@ -35,7 +37,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \ # This commit uses Boost 1.75 and libusb 1.0.24 git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \ bootstrap-vcpkg.bat + # Add custom UHD vcpkg triplet +COPY host/cmake/vcpkg/* c:/vcpkg/triplets/ RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \ - libusb:x64-windows \ - boost:x64-windows - + libusb:uhd-x64-windows-static-md \ + boost:uhd-x64-windows-static-md diff --git a/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile b/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile index 92afa8214..87806af1e 100644 --- a/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile +++ b/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile @@ -1,3 +1,5 @@ +# Docker build context must be uhd/uhddev root + # please follow docker best practices # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ @@ -35,6 +37,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \ # This commit uses Boost 1.75 and libusb 1.0.24 git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \ bootstrap-vcpkg.bat + # Add custom UHD vcpkg triplet +COPY host/cmake/vcpkg/* c:/vcpkg/triplets/ RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \ - libusb:x86-windows \ - boost:x86-windows + libusb:uhd-x86-windows-static-md \ + boost:uhd-x86-windows-static-md diff --git a/.ci/templates/job-get-latest-uhd-docker.yml b/.ci/templates/job-get-latest-uhd-docker.yml new file mode 100644 index 000000000..c90320d78 --- /dev/null +++ b/.ci/templates/job-get-latest-uhd-docker.yml @@ -0,0 +1,36 @@ +# Must include uhd-pipeline-vars.yml in top line pipeline to include this template +# Reads and sets uhd docker builder information to variables +# dockerBuildNumber, dockerImagePrefix, dockerImageMatrix +# Reference the data by calling: +# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ] +# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ] +# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ] +# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ] +jobs: +- job: get_latest_uhd_docker + displayName: Get Last Docker Build Version + pool: + name: Drivers-NIBuildFarm-RFMIBUILD + demands: sh + steps: + - checkout: none + - download: uhd_build_docker_container + displayName: Download Docker Image List + - script: | + cd $(Pipeline.Workspace)/uhd_build_docker_container/ + docker_build_number=$(sed '1q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName)) + docker_image_prefix=$(sed '2q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName)) + docker_image_matrix_lin=$(sed '3q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName)) + docker_image_matrix_win=$(sed '4q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName)) + echo "##vso[task.setvariable variable=dockerBuildNumber;isOutput=true;]$docker_build_number" + echo "##vso[task.setvariable variable=dockerImagePrefix;isOutput=true;]$docker_image_prefix" + echo "##vso[task.setvariable variable=dockerImageMatrixLin;isOutput=true;]$docker_image_matrix_lin" + echo "##vso[task.setvariable variable=dockerImageMatrixWin;isOutput=true;]$docker_image_matrix_win" + name: setDockerVar + - script: | + echo "Docker Build Number: " + echo $(setDockerVar.dockerBuildNumber) + echo $(setDockerVar.dockerImagePrefix) + echo $(setDockerVar.dockerImageMatrixLin) + echo $(setDockerVar.dockerImageMatrixWin) + name: echoDockerVar diff --git a/.ci/templates/job-uhd-build-src.yml b/.ci/templates/job-uhd-build-src.yml new file mode 100644 index 000000000..6e945d789 --- /dev/null +++ b/.ci/templates/job-uhd-build-src.yml @@ -0,0 +1,99 @@ +parameters: +- name: 'toolset' + type: string + values: + - make + - ninja + - msbuild + +jobs: +- job: build_ctest_and_upload_uhd_${{ parameters.toolset }} + displayName: ${{ parameters.toolset }}, ctest, and upload UHD + dependsOn: get_latest_uhd_docker + variables: + # Docker vars from job-get-latest-uhd-docker.yml + dockerBuildNumber: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ] + dockerImagePrefix: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ] + dockerImageMatrixLin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ] + dockerImageMatrixWin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ] + pool: + ${{ if eq(parameters.toolset, 'msbuild') }}: + name: de-dre-lab + ${{ if ne(parameters.toolset, 'msbuild') }}: + name: Drivers-NIBuildFarm-RFMIBUILD + demands: + - docker + - ${{ if eq(parameters.toolset, 'msbuild') }}: + - Agent.OS -equals Windows_NT + - ${{ if ne(parameters.toolset, 'msbuild') }}: + - Agent.OS -equals Linux + workspace: + clean: outputs + strategy: + ${{ if eq(parameters.toolset, 'msbuild') }}: + matrix: $[ variables.dockerImageMatrixWin ] + ${{ if ne(parameters.toolset, 'msbuild') }}: + matrix: $[ variables.dockerImageMatrixLin ] + container: + image: '$(dockerImagePrefix)$(DockerImageName):$(dockerBuildNumber)' + endpoint: 'Docker / Artifactory - rnd-builds-local' + steps: + - checkout: self + clean: true + + - download: uhd_build_docker_container + + - ${{ if eq(parameters.toolset, 'make') }}: + - template: steps-build-uhd-make.yml + parameters: + uhdSrcDir: $(Build.SourcesDirectory) + uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build + + - ${{ if eq(parameters.toolset, 'ninja') }}: + - template: steps-build-uhd-ninja.yml + parameters: + uhdSrcDir: $(Build.SourcesDirectory) + uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build + + - ${{ if eq(parameters.toolset, 'msbuild') }}: + - template: steps-build-uhd-msbuild.yml + parameters: + uhdSrcDir: $(Build.SourcesDirectory) + uhdBuildDir: $(Build.BinariesDirectory)\uhddev\build + cmakeCompiler: $(cmakeCompiler) + cmakeArch: $(cmakeArch) + vsArch: $(vsArch) + vsYear: $(vsYear) + + - task: CopyFiles@2 + inputs: + sourceFolder: $(Build.BinariesDirectory) + targetFolder: $(Build.ArtifactStagingDirectory) + displayName: Copy build files to artifact folder + + - task: CopyFiles@2 + inputs: + sourceFolder: $(Pipeline.Workspace)/uhd_build_docker_container/$(UHDPipelineDockerImageArtifactName) + targetFolder: $(Build.ArtifactStagingDirectory) + displayName: Copy builder container version to artifact folder + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: '$(dockerOSName)-${{ parameters.toolset }}' + displayName: Upload uhd build artifacts + + - task: CopyFiles@2 + inputs: + sourceFolder: $(Build.BinariesDirectory)/uhddev/ + contents: build/Testing/** + targetFolder: $(Common.TestResultsDirectory) + displayName: Copy ${{ parameters.toolset }} ctest results + + - task: PublishTestResults@2 + inputs: + testResultsFormat: cTest + testResultsFiles: $(Common.TestResultsDirectory)/build/Testing/**/Test.xml + failTaskOnFailedTests: true + testRunTitle: uhd ctest $(dockerOSName)-${{ parameters.toolset }} + displayName: Publish $(dockerOSName)-${{ parameters.toolset }} ctest results diff --git a/.ci/templates/job-uhd-hardware-tests.yml b/.ci/templates/job-uhd-hardware-tests.yml index a2ca9755d..c0c51288b 100644 --- a/.ci/templates/job-uhd-hardware-tests.yml +++ b/.ci/templates/job-uhd-hardware-tests.yml @@ -25,7 +25,7 @@ jobs: - checkout: self clean: true - - template: steps-build-uhd.yml + - template: steps-build-uhd-make.yml parameters: uhdSrcDir: $(Build.SourcesDirectory)/uhddev uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build diff --git a/.ci/templates/steps-build-uhd-make.yml b/.ci/templates/steps-build-uhd-make.yml new file mode 100644 index 000000000..22cb60cc9 --- /dev/null +++ b/.ci/templates/steps-build-uhd-make.yml @@ -0,0 +1,21 @@ +parameters: +- name: uhdSrcDir + type: string +- name: uhdBuildDir + type: string + +steps: +- script: | + mkdir -p ${{ parameters.uhdBuildDir }} + cd ${{ parameters.uhdBuildDir }} + cmake ${{ parameters.uhdSrcDir }}/host + displayName: cmake make UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + make -j$(nproc) + displayName: make UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + ctest --no-compress-output --output-on-failure -T test + continueOnError: true + displayName: ctest make UHD diff --git a/.ci/templates/steps-build-uhd-msbuild.yml b/.ci/templates/steps-build-uhd-msbuild.yml new file mode 100644 index 000000000..d92ae88b6 --- /dev/null +++ b/.ci/templates/steps-build-uhd-msbuild.yml @@ -0,0 +1,33 @@ +parameters: +- name: uhdSrcDir + type: string +- name: uhdBuildDir + type: string +- name: cmakeCompiler + type: string +- name: cmakeArch + type: string +- name: vsArch + type: string +- name: vsYear + type: string + +steps: +- script: | + mkdir ${{ parameters.uhdBuildDir }} + cd ${{ parameters.uhdBuildDir }} + cmake ${{ parameters.uhdSrcDir }}/host ^ + -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^ + -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%/scripts/buildsystems/vcpkg.cmake ^ + -G "$(cmakeCompiler)" -A $(cmakeArch) + displayName: cmake msbuild UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + call "C:\Program Files (x86)\Microsoft Visual Studio\$(vsYear)\Community\VC\Auxiliary\Build\vcvarsall.bat" $(vsArch) + msbuild.exe ALL_BUILD.vcxproj /p:configuration=release + displayName: msbuild UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + ctest --no-compress-output --output-on-failure -T test + continueOnError: true + displayName: ctest msbuild UHD diff --git a/.ci/templates/steps-build-uhd-ninja.yml b/.ci/templates/steps-build-uhd-ninja.yml new file mode 100644 index 000000000..f3ee6cfaa --- /dev/null +++ b/.ci/templates/steps-build-uhd-ninja.yml @@ -0,0 +1,21 @@ +parameters: +- name: uhdSrcDir + type: string +- name: uhdBuildDir + type: string + +steps: +- script: | + mkdir -p ${{ parameters.uhdBuildDir }} + cd ${{ parameters.uhdBuildDir }} + cmake -G Ninja ${{ parameters.uhdSrcDir }}/host + displayName: cmake ninja UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + ninja + displayName: ninja UHD +- script: | + cd ${{ parameters.uhdBuildDir }} + ctest --no-compress-output --output-on-failure -T test + continueOnError: true + displayName: ctest ninja UHD diff --git a/.ci/templates/steps-build-uhd.yml b/.ci/templates/steps-build-uhd.yml deleted file mode 100644 index 22cb60cc9..000000000 --- a/.ci/templates/steps-build-uhd.yml +++ /dev/null @@ -1,21 +0,0 @@ -parameters: -- name: uhdSrcDir - type: string -- name: uhdBuildDir - type: string - -steps: -- script: | - mkdir -p ${{ parameters.uhdBuildDir }} - cd ${{ parameters.uhdBuildDir }} - cmake ${{ parameters.uhdSrcDir }}/host - displayName: cmake make UHD -- script: | - cd ${{ parameters.uhdBuildDir }} - make -j$(nproc) - displayName: make UHD -- script: | - cd ${{ parameters.uhdBuildDir }} - ctest --no-compress-output --output-on-failure -T test - continueOnError: true - displayName: ctest make UHD diff --git a/.ci/uhd-build-docker-container.yml b/.ci/uhd-build-docker-container.yml index ab5cfae12..ad66fd486 100644 --- a/.ci/uhd-build-docker-container.yml +++ b/.ci/uhd-build-docker-container.yml @@ -116,6 +116,7 @@ jobs: containerRegistry: 'Docker / Artifactory - rnd-builds-local' repository: '$(dockerImagePrefix)$(dockerImageName)' tags: $(Build.BuildNumber) + buildContext: $(Build.SourcesDirectory) arguments: --no-cache --pull - task: Docker@2 displayName: Push Image @@ -148,6 +149,7 @@ jobs: mkdir uhddev\build cd uhddev\build cmake $(Build.SourcesDirectory)/host ^ + -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^ -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%\scripts\buildsystems\vcpkg.cmake ^ -G "$(cmakeCompiler)" -A $(cmakeArch) displayName: Test image with cmake uhd diff --git a/.ci/uhd-pipeline.yml b/.ci/uhd-pipeline.yml new file mode 100644 index 000000000..58db21be3 --- /dev/null +++ b/.ci/uhd-pipeline.yml @@ -0,0 +1,40 @@ +trigger: + branches: + include: + - master + - titanium-master + paths: + include: + - host + - .ci/uhd-pipeline.yml +pr: + branches: + include: + - master + - titanium-master + paths: + include: + - host + - .ci/uhd-pipeline.yml + +variables: +- template: uhd-pipeline-vars.yml + +resources: + pipelines: + - pipeline: uhd_build_docker_container + source: 'uhddev Build Docker Containers' + branch: master + +jobs: +- template: templates/job-get-latest-uhd-docker.yml + +- template: templates/job-uhd-build-src.yml + parameters: + toolset: make +- template: templates/job-uhd-build-src.yml + parameters: + toolset: ninja +- template: templates/job-uhd-build-src.yml + parameters: + toolset: msbuild -- cgit v1.2.3