ci: Refactor installers and add Windows support

This commit refactors the uhd pipeline to have a new installer stage.
This also adds Windows NSIS installers to pipelines.

Signed-off-by: Steven Koo <steven.koo@ni.com>

6 files changed, 183 insertions, 39 deletions

diff --git a/.ci/templates/job-uhd-build-installer.yml b/.ci/templates/job-uhd-build-installer.yml
new file mode 100644
index 000000000..e0d9dc9d3
--- /dev/null
+++ b/.ci/templates/job-uhd-build-installer.yml
@@ -0,0 +1,87 @@
+parameters:
+- name: 'toolset'
+  type: string
+  values:
+  - make
+  - msbuild
+  - ubuntu_deb
+- name: 'installer'
+  type: string
+  values:
+  - rpm # make
+  - tar # make
+  - nsis # msbuild
+  - deb # ubuntu_deb
+
+jobs:
+- job: build_uhd_installer_${{ parameters.toolset }}
+  displayName: ${{ parameters.toolset }} UHD installer
+  dependsOn: get_latest_uhd_docker
+  variables:
+    # Docker vars from job-get-latest-uhd-docker.yml
+    dockerBuildNumber: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ]
+    dockerImagePrefix: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ]
+    dockerImageMatrixLin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ]
+    dockerImageMatrixWin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ]
+    dockerImageMatrixUbuntuDeb: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixUbuntuDeb'] ]
+  pool:
+    ${{ if eq(parameters.toolset, 'msbuild') }}:
+      name: de-dre-lab
+    ${{ if ne(parameters.toolset, 'msbuild') }}:
+      name: Drivers-NIBuildFarm-RFMIBUILD
+    demands:
+      - docker
+      - ${{ if eq(parameters.toolset, 'msbuild') }}:
+        - Agent.OS -equals Windows_NT
+      - ${{ if ne(parameters.toolset, 'msbuild') }}:
+        - Agent.OS -equals Linux
+  workspace:
+    clean: outputs
+  strategy:
+    ${{ if and(eq(parameters.toolset, 'make'), eq(parameters.installer, 'tarball')) }}:
+      matrix: $[ variables.dockerImageMatrixLin ]
+    ${{ if and(eq(parameters.toolset, 'msbuild'), eq(parameters.installer, 'nsis')) }}:
+      matrix: $[ variables.dockerImageMatrixWin ]
+    ${{ if and(eq(parameters.toolset, 'ubuntu_deb'), eq(parameters.installer, 'deb')) }}:
+      matrix: $[ variables.dockerImageMatrixUbuntuDeb ]
+  container:
+    image: '$(dockerImagePrefix)$(DockerImageName):$(dockerBuildNumber)'
+    endpoint: 'Docker / Artifactory - rnd-builds-local'
+    ${{ if eq(parameters.toolset, 'ubuntu_deb') }}:
+      #pbuilder requires access to /proc
+      options: --privileged
+  steps:
+  - checkout: self
+    clean: true
+
+  - download: uhd_build_docker_container
+  - ${{ if eq(parameters.toolset, 'ubuntu_deb') }}:
+    - template: steps-build-uhd-installer-ubuntu-deb.yml
+      parameters:
+        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdInstallerDir: $(Build.BinariesDirectory)/uhddev-installer
+        ubuntuReleaseName: $(ubuntuReleaseName)
+  - ${{ if ne(parameters.toolset, 'ubuntu_deb') }}:
+    - download: current
+      artifact: $(dockerOSName)-${{ parameters.toolset }}
+      displayName: Download pipeline artifact $(dockerOSName)-${{ parameters.toolset }}
+    - task: ExtractFiles@1
+      inputs:
+        archiveFilePatterns: $(Pipeline.Workspace)/$(dockerOSName)-${{ parameters.toolset }}/$(dockerOSName)-${{ parameters.toolset }}.tar.gz
+        destinationFolder: $(Build.BinariesDirectory)
+        cleanDestinationFolder: true
+  - ${{ if and(eq(parameters.toolset, 'msbuild'), eq(parameters.installer, 'nsis')) }}:
+    - template: steps-build-uhd-installer-msbuild-nsis.yml
+      parameters:
+        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdBuildDir: $(Build.BinariesDirectory)\\uhddev\\build
+        uhdInstallerDir: $(Build.BinariesDirectory)\\uhddev-installer
+        cmakeCompiler: $(cmakeCompiler)
+        cmakeArch: $(cmakeArch)
+        vsArch: $(vsArch)
+        vsYear: $(vsYear)
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Build.BinariesDirectory)/uhddev-installer'
+      artifactName: '$(dockerOSName)-${{ parameters.toolset }}-${{ parameters.installer }}-installer'
+    displayName: Upload uhd installer artifact
diff --git a/.ci/templates/job-uhd-build-src.yml b/.ci/templates/job-uhd-build-src.yml
index 25f8b4897..ea3abbede 100644
--- a/.ci/templates/job-uhd-build-src.yml
+++ b/.ci/templates/job-uhd-build-src.yml
@@ -7,11 +7,12 @@ parameters:
   - make_trace
   - ninja
   - msbuild
-  - ubuntu_deb
 - name: 'custom_boost_version_url'
   type: string
   default: 'https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.bz2'
-
+- name: release_binaries
+  type: boolean
+  default: false
 
 jobs:
 - job: build_ctest_and_upload_uhd_${{ parameters.toolset }}
@@ -42,14 +43,9 @@ jobs:
       matrix: $[ variables.dockerImageMatrixWin ]
     ${{ if or(eq(parameters.toolset, 'make'), eq(parameters.toolset, 'make_trace'), eq(parameters.toolset, 'ninja'), eq(parameters.toolset, 'make_custom_boost_version')) }}:
       matrix: $[ variables.dockerImageMatrixLin ]
-    ${{ if eq(parameters.toolset, 'ubuntu_deb') }}:
-      matrix: $[ variables.dockerImageMatrixUbuntuDeb ]
   container:
     image: '$(dockerImagePrefix)$(DockerImageName):$(dockerBuildNumber)'
     endpoint: 'Docker / Artifactory - rnd-builds-local'
-    ${{ if eq(parameters.toolset, 'ubuntu_deb') }}:
-      #pbuilder requires access to /proc
-      options: --privileged
   steps:
   - checkout: self
     clean: true
@@ -59,39 +55,34 @@ jobs:
   - ${{ if eq(parameters.toolset, 'make') }}:
     - template: steps-build-uhd-make.yml
       parameters:
-        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdSrcDir: ../../../s
         uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
 
   - ${{ if eq(parameters.toolset, 'make_trace') }}:
     - template: steps-build-uhd-make.yml
       parameters:
-        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdSrcDir: ../../../s
         uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
         uhdEnableTraceLog: true
 
   - ${{ if eq(parameters.toolset, 'ninja') }}:
     - template: steps-build-uhd-ninja.yml
       parameters:
-        uhdSrcDir: $(Build.SourcesDirectory)
+        uhdSrcDir: ../../../s
         uhdBuildDir: $(Build.BinariesDirectory)/uhddev/build
 
   - ${{ if eq(parameters.toolset, 'msbuild') }}:
     - template: steps-build-uhd-msbuild.yml
       parameters:
-        uhdSrcDir: $(Build.SourcesDirectory)
-        uhdBuildDir: $(Build.BinariesDirectory)\uhddev\build
+        uhdSrcDir: ../../../s
+        uhdBuildDir: $(Build.BinariesDirectory)\\uhddev\\build
+        uhdImageDir: $(Build.BinariesDirectory)\\uhd-images
+        uhdReleaseBinaries: ${{ parameters.release_binaries }}
         cmakeCompiler: $(cmakeCompiler)
         cmakeArch: $(cmakeArch)
         vsArch: $(vsArch)
         vsYear: $(vsYear)
 
-  - ${{ if eq(parameters.toolset, 'ubuntu_deb') }}:
-    - template: steps-build-uhd-ubuntu-deb.yml
-      parameters:
-        uhdSrcDir: $(Build.SourcesDirectory)
-        uhdBuildDir: $(Build.BinariesDirectory)/uhddev
-        ubuntuReleaseName: $(ubuntuReleaseName)
-
   - ${{ if eq(parameters.toolset, 'make_custom_boost_version') }}:
     - template: steps-build-uhd-make.yml
       parameters:
@@ -122,18 +113,17 @@ jobs:
       artifactName: '$(dockerOSName)-${{ parameters.toolset }}'
     displayName: Upload uhd build artifacts
 
-  - ${{ if ne(parameters.toolset, 'ubuntu_deb') }}:
-    - task: CopyFiles@2
-      inputs:
-        sourceFolder: $(Build.BinariesDirectory)/uhddev/
-        contents: build/Testing/**
-        targetFolder: $(Common.TestResultsDirectory)
-      displayName: Copy ${{ parameters.toolset }} ctest results
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)/uhddev/
+      contents: build/Testing/**
+      targetFolder: $(Common.TestResultsDirectory)
+    displayName: Copy ${{ parameters.toolset }} ctest results
 
-    - task: PublishTestResults@2
-      inputs:
-        testResultsFormat: cTest
-        testResultsFiles: $(Common.TestResultsDirectory)/build/Testing/**/Test.xml
-        failTaskOnFailedTests: true
-        testRunTitle: uhd ctest $(dockerOSName)-${{ parameters.toolset }}
-      displayName: Publish $(dockerOSName)-${{ parameters.toolset }} ctest results
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: cTest
+      testResultsFiles: $(Common.TestResultsDirectory)/build/Testing/**/Test.xml
+      failTaskOnFailedTests: true
+      testRunTitle: uhd ctest $(dockerOSName)-${{ parameters.toolset }}
+    displayName: Publish $(dockerOSName)-${{ parameters.toolset }} ctest results
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index f7d705fed..f5ff5c227 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -11,6 +11,9 @@ parameters:
 - name: custom_boost_version_url
   type: string
   default: 'https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.bz2'
+- name: release_binaries
+  type: boolean
+  default: false
 
 variables:
 - template: ../uhd-pipeline-vars.yml
@@ -38,15 +41,27 @@ stages:
   - template: job-uhd-build-src.yml
     parameters:
       toolset: msbuild
-  - template: job-uhd-build-src.yml
-    parameters:
-      toolset: ubuntu_deb
+      release_binaries: ${{ parameters.release_binaries }}
+
   - ${{ if parameters.custom_boost_version }}:
     - template: job-uhd-build-src.yml
       parameters:
         toolset: make_custom_boost_version
         custom_boost_version_url: ${{ parameters.custom_boost_version_url }}
 
+- stage: build_uhd_installer_stage
+  displayName: Build UHD Installers
+  dependsOn: build_uhd_stage
+  jobs:
+    - template: job-get-latest-uhd-docker.yml
+    - template: job-uhd-build-installer.yml
+      parameters:
+        toolset: ubuntu_deb
+        installer: deb
+    - template: job-uhd-build-installer.yml
+      parameters:
+        toolset: msbuild
+        installer: nsis
 
 - stage: test_uhd_stage
   displayName: Test UHD
diff --git a/.ci/templates/steps-build-uhd-installer-msbuild-nsis.yml b/.ci/templates/steps-build-uhd-installer-msbuild-nsis.yml
new file mode 100644
index 000000000..ba388e5f6
--- /dev/null
+++ b/.ci/templates/steps-build-uhd-installer-msbuild-nsis.yml
@@ -0,0 +1,29 @@
+parameters:
+- name: uhdSrcDir
+  type: string
+- name: uhdBuildDir
+  type: string
+- name: uhdInstallerDir
+  type: string
+- name: cmakeCompiler
+  type: string
+- name: cmakeArch
+  type: string
+- name: vsArch
+  type: string
+- name: vsYear
+  type: string
+
+steps:
+- script: |
+    cd ${{ parameters.uhdBuildDir }}
+    call "C:\Program Files (x86)\Microsoft Visual Studio\$(vsYear)\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" $(vsArch)
+    msbuild.exe package.vcxproj /p:configuration=release
+  displayName: msbuild package
+- task: CopyFiles@2
+  inputs:
+    sourceFolder: ${{ parameters.uhdBuildDir }}
+    contents: 'uhd_*.exe'
+    targetFolder: ${{ parameters.uhdInstallerDir }}
+    cleanTargetFolder: true
+  displayName: Copy msbuild installer
diff --git a/.ci/templates/steps-build-uhd-ubuntu-deb.yml b/.ci/templates/steps-build-uhd-installer-ubuntu-deb.yml
index 6ad6680f2..98a3ac556 100644
--- a/.ci/templates/steps-build-uhd-ubuntu-deb.yml
+++ b/.ci/templates/steps-build-uhd-installer-ubuntu-deb.yml
@@ -1,7 +1,7 @@
 parameters:
 - name: uhdSrcDir
   type: string
-- name: uhdBuildDir
+- name: uhdInstallerDir
   type: string
 - name: ubuntuReleaseName
   type: string
@@ -11,7 +11,7 @@ steps:
     sudo pbuilder create --debootstrapopts --variant=buildd --distribution ${{ parameters.ubuntuReleaseName }}
   displayName: Setup pbuilder
 - script: |
-    mkdir -p ${{ parameters.uhdBuildDir }}
+    mkdir -p ${{ parameters.uhdInstallerDir }}
     cd ${{ parameters.uhdSrcDir }}
-    python3 tools/debs/uhd_ubuntu_deb.py ${{ parameters.ubuntuReleaseName }} --buildpath ${{ parameters.uhdBuildDir }}
+    python3 tools/debs/uhd_ubuntu_deb.py ${{ parameters.ubuntuReleaseName }} --buildpath ${{ parameters.uhdInstallerDir }}
   displayName: Build dsc and debs
diff --git a/.ci/templates/steps-build-uhd-msbuild.yml b/.ci/templates/steps-build-uhd-msbuild.yml
index 3b2c6120d..31bb1dfaa 100644
--- a/.ci/templates/steps-build-uhd-msbuild.yml
+++ b/.ci/templates/steps-build-uhd-msbuild.yml
@@ -3,6 +3,9 @@ parameters:
   type: string
 - name: uhdBuildDir
   type: string
+- name: uhdImageDir
+  type: string
+- name: uhdReleaseBinaries
 - name: cmakeCompiler
   type: string
 - name: cmakeArch
@@ -18,7 +21,27 @@ steps:
     cd ${{ parameters.uhdBuildDir }}
     cmake ${{ parameters.uhdSrcDir }}/host ^
       -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^
+      -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%/scripts/buildsystems/vcpkg.cmake
+    mkdir ${{ parameters.uhdImageDir }}
+    python ${{ parameters.uhdBuildDir }}/utils/uhd_images_downloader.py -i ${{ parameters.uhdImageDir }}
+    cd $(Pipeline.Workspace)
+    rmdir ${{ parameters.uhdBuildDir }} /s /q
+  displayName: Download uhd-images
+- script: |
+    mkdir ${{ parameters.uhdBuildDir }}
+    cd ${{ parameters.uhdBuildDir }}
+    if "${{ parameters.uhdReleaseBinaries }}" == "True" (
+      echo "Setting Release Mode"
+      SET UHD_CI_CMAKE_OPTIONS="-DUHD_RELEASE_MODE=release %UHD_CI_CMAKE_OPTIONS%"
+    )
+    cmake ${{ parameters.uhdSrcDir }}/host ^
+      -DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^
       -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%/scripts/buildsystems/vcpkg.cmake ^
+      -DUHD_IMAGES_SRC_DIR="${{ parameters.uhdImageDir }}" ^
+      -DUHD_IMAGES_DIR="C:\\Program Files (x86)\\UHD\\share\\uhd\\images,C:\\Program Files\\UHD\\share\\uhd\\images" ^
+      -DSPECIFY_MSVC_VERSION=ON ^
+      -DENABLE_DOXYGEN_SHORTNAMES=ON ^
+      %UHD_CI_CMAKE_OPTIONS% ^
       -G "$(cmakeCompiler)" -A $(cmakeArch)
   displayName: cmake msbuild UHD
 - script: |