ci: Build uhd with Pipelines

This commit adds the infrastructure to build uhd on both Linux and
Windows using Pipelines. Using the docker images generated by the
build docker pipeline, this runs an AzDO container job to build uhd
artifacts. It then stores the artifacts to be used in later pipelines or
stages. This change supports make and ninja on Linux and msbuild
with VS2017/VS2019 x86/x64 on Windows.

Signed-off-by: Steven Koo <steven.koo@ni.com>

12 files changed, 256 insertions, 11 deletions

diff --git a/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile b/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile
index d608ef4c4..12e198139 100644
--- a/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile
+++ b/.ci/docker/uhd-builder-vs2017-v141-x64.Dockerfile
@@ -1,3 +1,5 @@
+# Docker build context must be uhd/uhddev root

+

 # please follow docker best practices

 # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/

 

@@ -36,7 +38,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \
     # This commit uses Boost 1.75 and libusb 1.0.24

     git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \

     bootstrap-vcpkg.bat

+    # Add custom UHD vcpkg triplet

+COPY host/cmake/vcpkg/* c:/vcpkg/triplets/

 RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \

-    libusb:x64-windows \

-    boost:x64-windows

-

+    libusb:uhd-x64-windows-static-md \

+    boost:uhd-x64-windows-static-md

diff --git a/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile b/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile
index f4cb509fe..6ef77132d 100644
--- a/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile
+++ b/.ci/docker/uhd-builder-vs2017-v141-x86.Dockerfile
@@ -1,3 +1,5 @@
+# Docker build context must be uhd/uhddev root

+

 # please follow docker best practices

 # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/

 

@@ -36,6 +38,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \
     # This commit uses Boost 1.75 and libusb 1.0.24

     git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \

     bootstrap-vcpkg.bat

+    # Add custom UHD vcpkg triplet

+COPY host/cmake/vcpkg/* c:/vcpkg/triplets/

 RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \

-    libusb:x86-windows \

-    boost:x86-windows

+    libusb:uhd-x86-windows-static-md \

+    boost:uhd-x86-windows-static-md

diff --git a/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile b/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile
index eb8aedcf6..d10b87cac 100644
--- a/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile
+++ b/.ci/docker/uhd-builder-vs2019-v142-x64.Dockerfile
@@ -1,3 +1,5 @@
+# Docker build context must be uhd/uhddev root

+

 # please follow docker best practices

 # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/

 

@@ -35,7 +37,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \
     # This commit uses Boost 1.75 and libusb 1.0.24

     git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \

     bootstrap-vcpkg.bat

+    # Add custom UHD vcpkg triplet

+COPY host/cmake/vcpkg/* c:/vcpkg/triplets/

 RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \

-    libusb:x64-windows \

-    boost:x64-windows

-

+    libusb:uhd-x64-windows-static-md \

+    boost:uhd-x64-windows-static-md

diff --git a/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile b/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile
index 92afa8214..87806af1e 100644
--- a/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile
+++ b/.ci/docker/uhd-builder-vs2019-v142-x86.Dockerfile
@@ -1,3 +1,5 @@
+# Docker build context must be uhd/uhddev root

+

 # please follow docker best practices

 # https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/

 

@@ -35,6 +37,8 @@ RUN git clone https://github.com/microsoft/vcpkg %VCPKG_INSTALL_DIR% && \
     # This commit uses Boost 1.75 and libusb 1.0.24

     git checkout 3426db05b996481ca31e95fff3734cf23e0f51bc && \

     bootstrap-vcpkg.bat

+    # Add custom UHD vcpkg triplet

+COPY host/cmake/vcpkg/* c:/vcpkg/triplets/

 RUN cd %VCPKG_INSTALL_DIR% && vcpkg install --clean-after-build \

-    libusb:x86-windows \

-    boost:x86-windows

+    libusb:uhd-x86-windows-static-md \

+    boost:uhd-x86-windows-static-md

diff --git a/.ci/templates/job-get-latest-uhd-docker.yml b/.ci/templates/job-get-latest-uhd-docker.yml
new file mode 100644
index 000000000..c90320d78
--- /dev/null
+++ b/.ci/templates/job-get-latest-uhd-docker.yml
@@ -0,0 +1,36 @@
+# Must include uhd-pipeline-vars.yml in top line pipeline to include this template
+# Reads and sets uhd docker builder information to variables
+# dockerBuildNumber, dockerImagePrefix, dockerImageMatrix
+# Reference the data by calling:
+# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ]
+# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ]
+# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ]
+# $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ]
+jobs:
+- job: get_latest_uhd_docker
+  displayName: Get Last Docker Build Version
+  pool:
+    name: Drivers-NIBuildFarm-RFMIBUILD
+    demands: sh
+  steps:
+  - checkout: none
+  - download: uhd_build_docker_container
+    displayName: Download Docker Image List
+  - script: |
+      cd $(Pipeline.Workspace)/uhd_build_docker_container/
+      docker_build_number=$(sed '1q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName))
+      docker_image_prefix=$(sed '2q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName))
+      docker_image_matrix_lin=$(sed '3q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName))
+      docker_image_matrix_win=$(sed '4q;d' $(UHDPipelineDockerImageArtifactName)/$(UHDPipelineDockerImageArtifactName))
+      echo "##vso[task.setvariable variable=dockerBuildNumber;isOutput=true;]$docker_build_number"
+      echo "##vso[task.setvariable variable=dockerImagePrefix;isOutput=true;]$docker_image_prefix"
+      echo "##vso[task.setvariable variable=dockerImageMatrixLin;isOutput=true;]$docker_image_matrix_lin"
+      echo "##vso[task.setvariable variable=dockerImageMatrixWin;isOutput=true;]$docker_image_matrix_win"
+    name: setDockerVar
+  - script: |
+      echo "Docker Build Number: "
+      echo $(setDockerVar.dockerBuildNumber)
+      echo $(setDockerVar.dockerImagePrefix)
+      echo $(setDockerVar.dockerImageMatrixLin)
+      echo $(setDockerVar.dockerImageMatrixWin)
+    name: echoDockerVar
diff --git a/.ci/templates/job-uhd-build-src.yml b/.ci/templates/job-uhd-build-src.yml
new file mode 100644
index 000000000..6e945d789
--- /dev/null
+++ b/.ci/templates/job-uhd-build-src.yml
@@ -0,0 +1,99 @@
+parameters:
+- name: 'toolset'
+  type: string
+  values:
+  - make
+  - ninja
+  - msbuild
+
+jobs:
+- job: build_ctest_and_upload_uhd_${{ parameters.toolset }}
+  displayName: ${{ parameters.toolset }}, ctest, and upload UHD
+  dependsOn: get_latest_uhd_docker
+  variables:
+    # Docker vars from job-get-latest-uhd-docker.yml
+    dockerBuildNumber: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ]
+    dockerImagePrefix: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ]
+    dockerImageMatrixLin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ]
+    dockerImageMatrixWin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ]
+  pool:
+    ${{ if eq(parameters.toolset, 'msbuild') }}:
+      name: de-dre-lab
+    ${{ if ne(parameters.toolset, 'msbuild') }}:
+      name: Drivers-NIBuildFarm-RFMIBUILD
+    demands:
+      - docker
+      - ${{ if eq(parameters.toolset, 'msbuild') }}:
+        - Agent.OS -equals Windows_NT
+      - ${{ if ne(parameters.toolset, 'msbuild') }}:
+        - Agent.OS -equals Linux
+  workspace:
+    clean: outputs
+  strategy:
+    ${{ if eq(parameters.toolset, 'msbuild') }}:
+      matrix: $[ variables.dockerImageMatrixWin ]
+    ${{ if ne(parameters.toolset, 'msbuild') }}:
+      matrix: $[ variables.dockerImageMatrixLin ]
+  container:
+    image: '$(dockerImagePrefix)$(DockerImageName):$(dockerBuildNumber)'
+    endpoint: 'Docker / Artifactory - rnd-builds-local'
+  steps:
+  - checkout: self
+    clean: true
+
+  - download: uhd_build_docker_container
+
+  - ${{ if eq(parameters.toolset, 'make') }}:
+    - template: steps-build-uhd-make.yml
+      parameters:
+        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
+
+  - ${{ if eq(parameters.toolset, 'ninja') }}:
+    - template: steps-build-uhd-ninja.yml
+      parameters:
+        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
+
+  - ${{ if eq(parameters.toolset, 'msbuild') }}:
+    - template: steps-build-uhd-msbuild.yml
+      parameters:
+        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdBuildDir: $(Build.BinariesDirectory)\uhddev\build
+        cmakeCompiler: $(cmakeCompiler)
+        cmakeArch: $(cmakeArch)
+        vsArch: $(vsArch)
+        vsYear: $(vsYear)
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: Copy build files to artifact folder
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Pipeline.Workspace)/uhd_build_docker_container/$(UHDPipelineDockerImageArtifactName)
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName:  Copy builder container version to artifact folder
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+      artifactName: '$(dockerOSName)-${{ parameters.toolset }}'
+    displayName: Upload uhd build artifacts
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)/uhddev/
+      contents: build/Testing/**
+      targetFolder: $(Common.TestResultsDirectory)
+    displayName: Copy ${{ parameters.toolset }} ctest results
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: cTest
+      testResultsFiles: $(Common.TestResultsDirectory)/build/Testing/**/Test.xml
+      failTaskOnFailedTests: true
+      testRunTitle: uhd ctest $(dockerOSName)-${{ parameters.toolset }}
+    displayName: Publish $(dockerOSName)-${{ parameters.toolset }} ctest results
diff --git a/.ci/templates/job-uhd-hardware-tests.yml b/.ci/templates/job-uhd-hardware-tests.yml
index a2ca9755d..c0c51288b 100644
--- a/.ci/templates/job-uhd-hardware-tests.yml
+++ b/.ci/templates/job-uhd-hardware-tests.yml
@@ -25,7 +25,7 @@ jobs:
   - checkout: self
     clean: true
 
-  - template: steps-build-uhd.yml
+  - template: steps-build-uhd-make.yml
     parameters:
       uhdSrcDir: $(Build.SourcesDirectory)/uhddev
       uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
diff --git a/.ci/templates/steps-build-uhd.yml b/.ci/templates/steps-build-uhd-make.yml
index 22cb60cc9..22cb60cc9 100644
--- a/.ci/templates/steps-build-uhd.yml
+++ b/.ci/templates/steps-build-uhd-make.yml
diff --git a/.ci/templates/steps-build-uhd-msbuild.yml b/.ci/templates/steps-build-uhd-msbuild.yml
new file mode 100644
index 000000000..d92ae88b6
--- /dev/null
+++ b/.ci/templates/steps-build-uhd-msbuild.yml
@@ -0,0 +1,33 @@
+parameters:
+- name: uhdSrcDir
+  type: string
+- name: uhdBuildDir
+  type: string
+- name: cmakeCompiler
+  type: string
+- name: cmakeArch
+  type: string
+- name: vsArch
+  type: string
+- name: vsYear
+  type: string
+
+steps:
+- script: |
+    mkdir ${{ parameters.uhdBuildDir }}
+    cd ${{ parameters.uhdBuildDir }}
+    cmake ${{ parameters.uhdSrcDir }}/host ^
+      -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^
+      -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%/scripts/buildsystems/vcpkg.cmake ^
+      -G "$(cmakeCompiler)" -A $(cmakeArch)
+  displayName: cmake msbuild UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    call "C:\Program Files (x86)\Microsoft Visual Studio\$(vsYear)\Community\VC\Auxiliary\Build\vcvarsall.bat" $(vsArch)
+    msbuild.exe ALL_BUILD.vcxproj /p:configuration=release
+  displayName: msbuild UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    ctest --no-compress-output --output-on-failure -T test
+  continueOnError: true
+  displayName: ctest msbuild UHD
diff --git a/.ci/templates/steps-build-uhd-ninja.yml b/.ci/templates/steps-build-uhd-ninja.yml
new file mode 100644
index 000000000..f3ee6cfaa
--- /dev/null
+++ b/.ci/templates/steps-build-uhd-ninja.yml
@@ -0,0 +1,21 @@
+parameters:
+- name: uhdSrcDir
+  type: string
+- name: uhdBuildDir
+  type: string
+
+steps:
+- script: |
+    mkdir -p ${{ parameters.uhdBuildDir }}
+    cd ${{ parameters.uhdBuildDir }}
+    cmake -G Ninja ${{ parameters.uhdSrcDir }}/host
+  displayName: cmake ninja UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    ninja
+  displayName: ninja UHD
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    ctest --no-compress-output --output-on-failure -T test
+  continueOnError: true
+  displayName: ctest ninja UHD
diff --git a/.ci/uhd-build-docker-container.yml b/.ci/uhd-build-docker-container.yml
index ab5cfae12..ad66fd486 100644
--- a/.ci/uhd-build-docker-container.yml
+++ b/.ci/uhd-build-docker-container.yml
@@ -116,6 +116,7 @@ jobs:
       containerRegistry: 'Docker / Artifactory - rnd-builds-local'
       repository: '$(dockerImagePrefix)$(dockerImageName)'
       tags: $(Build.BuildNumber)
+      buildContext: $(Build.SourcesDirectory)
       arguments: --no-cache --pull
   - task: Docker@2
     displayName: Push Image
@@ -148,6 +149,7 @@ jobs:
         mkdir uhddev\build
         cd uhddev\build
         cmake $(Build.SourcesDirectory)/host ^
+          -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^
           -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%\scripts\buildsystems\vcpkg.cmake ^
           -G "$(cmakeCompiler)" -A $(cmakeArch)
     displayName: Test image with cmake uhd
diff --git a/.ci/uhd-pipeline.yml b/.ci/uhd-pipeline.yml
new file mode 100644
index 000000000..58db21be3
--- /dev/null
+++ b/.ci/uhd-pipeline.yml
@@ -0,0 +1,40 @@
+trigger:
+  branches:
+    include:
+    - master
+    - titanium-master
+  paths:
+    include:
+    - host
+    - .ci/uhd-pipeline.yml
+pr:
+  branches:
+    include:
+    - master
+    - titanium-master
+  paths:
+    include:
+    - host
+    - .ci/uhd-pipeline.yml
+
+variables:
+- template: uhd-pipeline-vars.yml
+
+resources:
+  pipelines:
+  - pipeline: uhd_build_docker_container
+    source: 'uhddev Build Docker Containers'
+    branch: master
+
+jobs:
+- template: templates/job-get-latest-uhd-docker.yml
+
+- template: templates/job-uhd-build-src.yml
+  parameters:
+    toolset: make
+- template: templates/job-uhd-build-src.yml
+  parameters:
+    toolset: ninja
+- template: templates/job-uhd-build-src.yml
+  parameters:
+    toolset: msbuild