ci: Build gnuradio and gr-ettus

This change adds build support for gnuradio and gr-ettus.
After building gnuradio and gr-ettus 3.8 against the
uhddev commit, the job will upload the build artifact
for gnuradio and gr-ettus for use by downstream jobs,
like hardware testing.

Signed-off-by: Steven Koo <steven.koo@ni.com>

8 files changed, 209 insertions, 10 deletions

diff --git a/.ci/templates/job-gnuradio-grettus-build-src.yml b/.ci/templates/job-gnuradio-grettus-build-src.yml
new file mode 100644
index 000000000..90521f807
--- /dev/null
+++ b/.ci/templates/job-gnuradio-grettus-build-src.yml
@@ -0,0 +1,114 @@
+parameters:
+- name: 'toolset'
+  type: string
+  values:
+  - make
+
+jobs:
+- job: build_and_upload_gnuradio_grettus_${{ parameters.toolset }}
+  displayName: ${{ parameters.toolset }}, ctest, and upload gnuradio and grettus
+  dependsOn: get_latest_uhd_docker
+  variables:
+    # Docker vars from job-get-latest-uhd-docker.yml
+    dockerBuildNumber: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerBuildNumber'] ]
+    dockerImagePrefix: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImagePrefix'] ]
+    dockerImageMatrixLin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixLin'] ]
+    dockerImageMatrixWin: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixWin'] ]
+    dockerImageMatrixUbuntuDeb: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.dockerImageMatrixUbuntuDeb'] ]
+    macOSBuilders: $[ dependencies.get_latest_uhd_docker.outputs['setDockerVar.macOSBuilders'] ]
+  pool:
+    ${{ if eq(parameters.toolset, 'msbuild') }}:
+      name: de-dre-lab
+    ${{ if ne(parameters.toolset, 'msbuild') }}:
+      name: Drivers-NIBuildFarm-RFMIBUILD
+    demands:
+      - ${{ if eq(parameters.toolset, 'msbuild') }}:
+        - docker
+        - Agent.OS -equals Windows_NT
+      - ${{ if or(eq(parameters.toolset, 'make'), eq(parameters.toolset, 'make_trace'), eq(parameters.toolset, 'ninja'), eq(parameters.toolset, 'make_custom_boost_version')) }}:
+        - docker
+        - Agent.OS -equals Linux
+      - ${{ if eq(parameters.toolset, 'make_homebrew_macos') }}:
+        - Agent.OS -equals Darwin
+        - Agent.OSVersion -equals $(macOSVersionDemand)
+        - Agent.OSArchitecture -equals $(macOSArchDemand)
+  workspace:
+    clean: outputs
+  strategy:
+    ${{ if eq(parameters.toolset, 'msbuild') }}:
+      matrix: $[ variables.dockerImageMatrixWin ]
+    ${{ if or(eq(parameters.toolset, 'make'), eq(parameters.toolset, 'make_trace'), eq(parameters.toolset, 'ninja'), eq(parameters.toolset, 'make_custom_boost_version')) }}:
+      matrix: $[ variables.dockerImageMatrixLin ]
+    ${{ if eq(parameters.toolset, 'make_homebrew_macos') }}:
+      matrix: $[ variables.macOSBuilders ]
+  ${{ if ne(parameters.toolset, 'make_homebrew_macos') }}:
+    container:
+      image: '$(dockerImagePrefix)$(DockerImageName):$(dockerBuildNumber)'
+      endpoint: 'Docker / Artifactory - rnd-builds-local'
+  steps:
+  - checkout: self
+    clean: true
+  - checkout: gnuradio
+    clean: true
+    submodules: recursive
+  - checkout: gr-ettus
+    clean: true
+
+  - task: DeleteFiles@1
+    inputs:
+      SourceFolder: $(Agent.BuildDirectory)
+      Contents: |
+        uhddev-$(buildOSName)-${{ parameters.toolset }}
+    displayName: Delete uhddev artifacts folder
+
+  - download: uhd_build_docker_container
+
+  - download: current
+    artifact: uhddev-$(buildOSName)-${{ parameters.toolset }}
+    displayName: Download pipeline artifact uhddev-$(buildOSName)-${{ parameters.toolset }}
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: $(Pipeline.Workspace)/uhddev-$(buildOSName)-${{ parameters.toolset }}/uhddev-$(buildOSName)-${{ parameters.toolset }}.tar.gz
+      destinationFolder: $(Agent.BuildDirectory)/uhddev-$(buildOSName)-${{ parameters.toolset }}
+      cleanDestinationFolder: false
+    displayName: Extract uhddev-$(buildOSName)-${{ parameters.toolset }} artifacts
+
+  - ${{ if eq(parameters.toolset, 'make') }}:
+    - template: steps-build-gnuradio-make.yml
+      parameters:
+        gnuradioSrcDir: $(Build.SourcesDirectory)/gnuradio
+        gnuradioBuildDir: $(Build.BinariesDirectory)/gnuradio/build
+        gnuradioInstallPath: $(Build.BinariesDirectory)/gnuradio/build-installed
+        uhdInstalledArtifactPath: $(Agent.BuildDirectory)/uhddev-$(buildOSName)-${{ parameters.toolset }}/uhddev/build-installed
+
+  - ${{ if eq(parameters.toolset, 'make') }}:
+    - template: steps-build-gr-ettus-make.yml
+      parameters:
+        grettusSrcDir: $(Build.SourcesDirectory)/gr-ettusdev
+        grettusBuildDir: $(Build.BinariesDirectory)/gr-ettusdev/build
+        grettusInstallPath: $(Build.BinariesDirectory)/gnuradio/build-installed
+        uhdInstalledArtifactPath: $(Agent.BuildDirectory)/uhddev-$(buildOSName)-${{ parameters.toolset }}/uhddev/build-installed
+        gnuradioInstalledArtifactPath: $(Build.BinariesDirectory)/gnuradio/build-installed
+
+  - task: ArchiveFiles@2
+    inputs:
+      rootFolderOrFile: $(Build.BinariesDirectory)
+      includeRootFolder: false
+      archiveType: tar
+      tarCompression: gz
+      archiveFile: $(Build.ArtifactStagingDirectory)/gnuradio-grettus-$(buildOSName)-${{ parameters.toolset }}.tar.gz
+      replaceExistingArchive: true
+    displayName: Compress build files
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Pipeline.Workspace)/uhd_build_docker_container/$(UHDPipelineDockerImageArtifactName)
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName:  Copy builder container version to artifact folder
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+      artifactName: 'gnuradio-grettus-$(buildOSName)-${{ parameters.toolset }}'
+    displayName: Upload gnuradio and gr-ettus build artifacts
diff --git a/.ci/templates/job-uhd-build-installer.yml b/.ci/templates/job-uhd-build-installer.yml
index aff8dca5c..05df0fc39 100644
--- a/.ci/templates/job-uhd-build-installer.yml
+++ b/.ci/templates/job-uhd-build-installer.yml
@@ -63,11 +63,11 @@ jobs:
         ubuntuReleaseName: $(ubuntuReleaseName)
   - ${{ if ne(parameters.toolset, 'ubuntu_deb') }}:
     - download: current
-      artifact: $(buildOSName)-${{ parameters.toolset }}
-      displayName: Download pipeline artifact $(buildOSName)-${{ parameters.toolset }}
+      artifact: uhddev-$(buildOSName)-${{ parameters.toolset }}
+      displayName: Download pipeline artifact uhddev-$(buildOSName)-${{ parameters.toolset }}
     - task: ExtractFiles@1
       inputs:
-        archiveFilePatterns: $(Pipeline.Workspace)/$(buildOSName)-${{ parameters.toolset }}/$(buildOSName)-${{ parameters.toolset }}.tar.gz
+        archiveFilePatterns: $(Pipeline.Workspace)/uhddev-$(buildOSName)-${{ parameters.toolset }}/uhddev-$(buildOSName)-${{ parameters.toolset }}.tar.gz
         destinationFolder: $(Build.BinariesDirectory)
         cleanDestinationFolder: true
   - ${{ if and(eq(parameters.toolset, 'msbuild'), eq(parameters.installer, 'nsis')) }}:
@@ -83,5 +83,5 @@ jobs:
   - task: PublishPipelineArtifact@1
     inputs:
       targetPath: '$(Build.BinariesDirectory)/uhddev-installer'
-      artifactName: '$(buildOSName)-${{ parameters.toolset }}-${{ parameters.installer }}-installer'
+      artifactName: 'uhddev-$(buildOSName)-${{ parameters.toolset }}-${{ parameters.installer }}-installer'
     displayName: Upload uhd installer artifact
diff --git a/.ci/templates/job-uhd-build-src.yml b/.ci/templates/job-uhd-build-src.yml
index 49c65c1da..86b76db61 100644
--- a/.ci/templates/job-uhd-build-src.yml
+++ b/.ci/templates/job-uhd-build-src.yml
@@ -114,7 +114,7 @@ jobs:
       includeRootFolder: false
       archiveType: tar
       tarCompression: gz
-      archiveFile: $(Build.ArtifactStagingDirectory)/$(buildOSName)-${{ parameters.toolset }}.tar.gz
+      archiveFile: $(Build.ArtifactStagingDirectory)/uhddev-$(buildOSName)-${{ parameters.toolset }}.tar.gz
       replaceExistingArchive: true
     displayName: Compress build files
 
@@ -127,7 +127,7 @@ jobs:
   - task: PublishPipelineArtifact@1
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)'
-      artifactName: '$(buildOSName)-${{ parameters.toolset }}'
+      artifactName: 'uhddev-$(buildOSName)-${{ parameters.toolset }}'
     displayName: Upload uhd build artifacts
 
   - task: CopyFiles@2
diff --git a/.ci/templates/job-uhd-devtest.yml b/.ci/templates/job-uhd-devtest.yml
index 81733bd88..67b8cfeaf 100644
--- a/.ci/templates/job-uhd-devtest.yml
+++ b/.ci/templates/job-uhd-devtest.yml
@@ -42,11 +42,11 @@ jobs:
       sshPublicKey: '$(ettus_ssh_pubkey)'
       sshKeySecureFile: 'id_rsa.ettus'
   - download: current
-    artifact: ${{ parameters.testOS }}-${{ parameters.toolset }}
+    artifact: uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
     displayName: Download pipeline artifact ${{ parameters.testOS }}-${{ parameters.toolset }}
   - task: ExtractFiles@1
     inputs:
-      archiveFilePatterns: $(Pipeline.Workspace)/${{ parameters.testOS }}-${{ parameters.toolset }}/${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: true
   - script: |
diff --git a/.ci/templates/job-uhd-streaming-tests.yml b/.ci/templates/job-uhd-streaming-tests.yml
index 9a9b3168d..35eb7de54 100644
--- a/.ci/templates/job-uhd-streaming-tests.yml
+++ b/.ci/templates/job-uhd-streaming-tests.yml
@@ -36,11 +36,11 @@ jobs:
   - checkout: self
     clean: true
   - download: current
-    artifact: ${{ parameters.testOS }}-${{ parameters.toolset }}
+    artifact: uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}
     displayName: download pipeline artifact ${{ parameters.testOS }}-${{ parameters.toolset }}
   - task: ExtractFiles@1
     inputs:
-      archiveFilePatterns: $(Pipeline.Workspace)/${{ parameters.testOS }}-${{ parameters.toolset }}/${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
+      archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: true
   - script: |
diff --git a/.ci/templates/stages-uhd-pipeline.yml b/.ci/templates/stages-uhd-pipeline.yml
index 5e7e420a8..05e8f4cf3 100644
--- a/.ci/templates/stages-uhd-pipeline.yml
+++ b/.ci/templates/stages-uhd-pipeline.yml
@@ -67,6 +67,16 @@ resources:
     name: EttusResearch/meta-ettus-dev
     endpoint: EttusResearch
     ref: zeus-ci
+  - repository: gnuradio
+    type: github
+    name: gnuradio/gnuradio
+    endpoint: EttusResearch
+    ref: maint-3.8
+  - repository: gr-ettus
+    type: github
+    name: EttusResearch/gr-ettusdev
+    endpoint: EttusResearch
+    ref: maint-3.8-uhd4.0
 
 stages:
 - stage: build_uhd_stage_linux
@@ -155,6 +165,15 @@ stages:
           displayName: delete auto.conf
           condition: always()
 
+- stage: build_gnuradio_stage_linux
+  displayName: Build gnuradio Linux
+  dependsOn: build_uhd_stage_linux
+  jobs:
+  - template: job-get-latest-uhd-docker.yml
+  - template: job-gnuradio-grettus-build-src.yml
+    parameters:
+      toolset: make
+
 - stage: build_uhd_installer_stage_linux
   displayName: Build UHD Installers Linux
   dependsOn: build_uhd_stage_linux
diff --git a/.ci/templates/steps-build-gnuradio-make.yml b/.ci/templates/steps-build-gnuradio-make.yml
new file mode 100644
index 000000000..d8ca7cbee
--- /dev/null
+++ b/.ci/templates/steps-build-gnuradio-make.yml
@@ -0,0 +1,32 @@
+parameters:
+- name: gnuradioSrcDir
+  type: string
+- name: gnuradioBuildDir
+  type: string
+- name: gnuradioInstallPath
+  type: string
+- name: uhdInstalledArtifactPath
+  type: string
+
+steps:
+- script: |
+    mkdir -p ${{ parameters.gnuradioBuildDir }}
+    cd ${{ parameters.gnuradioBuildDir }}
+    mkdir -p ${{ parameters.gnuradioInstallPath }}
+    echo "Setting CMAKE_INSTALL_PREFIX to ${{ parameters.gnuradioInstallPath }}"
+    export UHD_CI_CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=${{ parameters.gnuradioInstallPath }}"
+    cmake \
+      $UHD_CI_CMAKE_OPTIONS \
+      -DCMAKE_PREFIX_PATH="${{ parameters.uhdInstalledArtifactPath }}" \
+      ${{ parameters.gnuradioSrcDir }}
+  displayName: cmake make gnuradio
+
+- script: |
+    cd ${{ parameters.gnuradioBuildDir }}
+    make -j$(nproc) -k
+  displayName: make gnuradio
+
+- script: |
+    cd ${{ parameters.gnuradioBuildDir }}
+    make install
+  displayName: Install gnuradio to gnuradioInstallPath
diff --git a/.ci/templates/steps-build-gr-ettus-make.yml b/.ci/templates/steps-build-gr-ettus-make.yml
new file mode 100644
index 000000000..a988eb23d
--- /dev/null
+++ b/.ci/templates/steps-build-gr-ettus-make.yml
@@ -0,0 +1,34 @@
+parameters:
+- name: grettusSrcDir
+  type: string
+- name: grettusBuildDir
+  type: string
+- name: grettusInstallPath
+  type: string
+- name: uhdInstalledArtifactPath
+  type: string
+- name: gnuradioInstalledArtifactPath
+  type: string
+
+steps:
+- script: |
+    mkdir -p ${{ parameters.grettusBuildDir }}
+    cd ${{ parameters.grettusBuildDir }}
+    mkdir -p ${{ parameters.grettusInstallPath }}
+    echo "Setting CMAKE_INSTALL_PREFIX to ${{ parameters.grettusInstallPath }}"
+    export UHD_CI_CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=${{ parameters.grettusInstallPath }}"
+    cmake \
+      $UHD_CI_CMAKE_OPTIONS \
+      -DCMAKE_PREFIX_PATH="${{ parameters.uhdInstalledArtifactPath }};${{ parameters.gnuradioInstalledArtifactPath }}" \
+      ${{ parameters.grettusSrcDir }}
+  displayName: cmake make gr-ettus
+
+- script: |
+    cd ${{ parameters.grettusBuildDir }}
+    make -j$(nproc) -k
+  displayName: make gr-ettus
+
+- script: |
+    cd ${{ parameters.grettusBuildDir }}
+    make install
+  displayName: Install gr-ettus to grettusInstallPath