fpga: ci: Add stages-based pipeline

13 files changed, 613 insertions, 271 deletions

diff --git a/fpga/.ci/fpga-pipeline-pr.yml b/fpga/.ci/fpga-pipeline-pr.yml
new file mode 100644
index 000000000..7f98c4fd6
--- /dev/null
+++ b/fpga/.ci/fpga-pipeline-pr.yml
@@ -0,0 +1,64 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This pipeline is used to build FPGAs and run testbenches for PRs.
+#
+#   See https://aka.ms/yaml for pipeline YAML documentation.
+#
+
+parameters:
+- name: run_testbenches
+  type: boolean
+  displayName: Run Testbenches
+  default: true
+- name: clean_ip_build
+  type: boolean
+  displayName: Clean IP Build
+  default: false
+- name: build_x410
+  type: boolean
+  displayName: Build X410
+  default: true
+- name: x410_targets_matrix
+  type: object
+  displayName: X410 Targets
+  default:
+    X410_XG_100:
+      target_name: X410_XG_100
+      timeout: 480
+    X410_X4_200:
+      target_name: X410_X4_200
+      timeout: 480
+    X410_CG_400:
+      target_name: X410_CG_400
+      timeout: 480
+- name: package_images
+  type: boolean
+  displayName: Package Images
+  default: false
+
+trigger: none
+
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - fpga/usrp3/lib
+    - fpga/usrp3/top/x400
+    - fpga/.ci
+
+extends:
+  template: templates/stages-fpga-pipeline.yml
+  parameters:
+    run_testbenches: ${{ parameters.run_testbenches }}
+    package_images: ${{ parameters.package_images }}
+    build_x410: ${{ parameters.build_x410 }}
+    x410_targets_matrix: ${{ parameters.x410_targets_matrix }}
+    publish_int_files: true
+    clean_ip_build: ${{ parameters.clean_ip_build }}
diff --git a/fpga/.ci/fpga-pipeline.yml b/fpga/.ci/fpga-pipeline.yml
new file mode 100644
index 000000000..9bf6d78f4
--- /dev/null
+++ b/fpga/.ci/fpga-pipeline.yml
@@ -0,0 +1,60 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This pipeline is used to build FPGAs and run testbenches as CI
+#   from commits in master.
+#
+#   See https://aka.ms/yaml for pipeline YAML documentation.
+#
+
+parameters:
+- name: run_testbenches
+  type: boolean
+  displayName: Run Testbenches
+  default: false
+- name: clean_ip_build
+  type: boolean
+  displayName: Clean IP Build
+  default: true
+- name: build_x410
+  type: boolean
+  displayName: Build X410
+  default: true
+- name: package_images
+  type: boolean
+  displayName: Package Images
+  default: true
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - fpga/usrp3/lib
+    - fpga/usrp3/top/x400
+    - fpga/.ci
+
+pr: none
+
+extends:
+  template: templates/stages-fpga-pipeline.yml
+  parameters:
+    run_testbenches: ${{ parameters.run_testbenches }}
+    package_images: ${{ parameters.package_images }}
+    build_x410: ${{ parameters.build_x410 }}
+    # These targets are shipped and included in the binaries package.
+    x410_targets_matrix:
+      X410_X4_200:
+        target_name: X410_X4_200
+        timeout: 480
+      X410_CG_400:
+        target_name: X410_CG_400
+        timeout: 480
+    publish_int_files: false
+    clean_ip_build: ${{ parameters.clean_ip_build }}
diff --git a/fpga/.ci/oss_testbenches.yml b/fpga/.ci/oss_testbenches.yml
deleted file mode 100644
index f4a01edbc..000000000
--- a/fpga/.ci/oss_testbenches.yml
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Copyright 2021 Ettus Research, a National Instruments Brand
-#
-# SPDX-License-Identifier: LGPL-3.0-or-later
-#
-# Description:
-#
-#   This pipeline is used to run all the testbenches using ModelSim.
-#
-
-trigger:
-- none
-
-pr:
-  branches:
-    include:
-    - master
-  paths:
-    include:
-    - fpga/usrp3/lib
-    - fpga/usrp3/tools
-    - fpga/usrp3/top
-    - fpga/usrp3/sim
-    - fpga/.ci
-
-resources:
-  repositories:
-  - repository: hwtools
-    type: git
-    ref: main
-    name: DevCentral/hwtools
-
-name:
-jobs:
-- job:
-  displayName: "ModelSim Simulation"
-  timeoutInMinutes: 360
-  pool:
-    name: Hardware
-  steps:
-    - checkout: self
-      clean: true
-      persistCredentials: true
-
-    - checkout: hwtools
-      clean: true
-      path: s/hwtools/head
-      persistCredentials: true
-
-    - bash: |
-        set -e
-
-        echo "---- Set environment variables ----"
-        export path_hwtools=$(Pipeline.Workspace)/s/hwtools/head/setup
-        export PATH=$path_hwtools:$PATH
-
-        echo "---- Run hwsetup ----"
-        # This script sets the XILINX_VIVADO, MODELSIM, and LIB_BASE_PATH
-        # variables based on the agent's configuration so we can find the EDA
-        # tools.
-        pushd ../.ci/hwtools
-        source hwsetup.sh
-        popd
-
-        echo "---- Run setupenv ----"
-        export MSIM_VIV_COMPLIBDIR=$LIB_BASE_PATH/vivado/2019.1.1/modelsim_SE-64_2020
-        source ./top/x300/setupenv.sh --vivado-path $(dirname $XILINX_VIVADO) --modelsim-path $(dirname $MODELSIM)
-
-        echo "---- Run testbenches ----"
-        pushd ./tools/utils
-        python3 ./run_testbenches.py --logged --simulator modelsim --excludes=modelsim.excludes -j2 run
-        popd
-      workingDirectory: uhddev/fpga/usrp3
-      displayName: "Run Testbenches"
diff --git a/fpga/.ci/scripts/refresh_ip.sh b/fpga/.ci/scripts/refresh_ip.sh
new file mode 100644
index 000000000..f4e37bf5b
--- /dev/null
+++ b/fpga/.ci/scripts/refresh_ip.sh
@@ -0,0 +1,38 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This script first refreshes a given build IP directory to the current time
+#   (assume all the IP files within that directory were just built) and then
+#   the script touches all the files that changed between the current repo's
+#   working tree and a given hash (used to build IP).
+#   This allows the user to download pre-built IP (e.g. cached) and re-build
+#   only those IP components that changed in between the cached commit and the
+#   current working tree.
+#
+# Arguments:
+#
+#   $1 = absolute path to build-ip/ directory
+#   $2 = git hash used to build IP in $1
+#
+# Example:
+#
+#   bash .refresh_ip.sh `realpath path/to/build-ip` 1234abc
+#
+
+# # Step 1: refresh build-ip/ directory to current time
+echo "[refresh_ip.sh] Resetting timestamp for files in $1"
+find $1 -type f -exec touch --no-create {} +
+
+# # Step 2: determine all files that changed and update their time (dirty)
+echo "[refresh_ip.sh] Differences between $2 and working tree (marked dirty):"
+git diff --line-prefix=`git rev-parse --show-toplevel`/ --name-only $2 | \
+  while read -r line; do \
+    if [[ $line == *"ip/"* ]]; then \
+      echo "    $line"; \
+      touch --no-create $line; \
+    fi \
+  done
diff --git a/fpga/.ci/scripts/run_setup.sh b/fpga/.ci/scripts/run_setup.sh
new file mode 100644
index 000000000..c70fd5702
--- /dev/null
+++ b/fpga/.ci/scripts/run_setup.sh
@@ -0,0 +1,43 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   Script to run NI hwsetup and UHD setupenv for the target build. Always
+#   source this file.
+#
+#   Arguments:
+#
+#     $1 = Directory where setupenv is located
+#
+#  Example:
+#
+#     source run_setup.sh ./usrp3/top/x400
+#
+
+set -e
+
+echo "---- Set environment variables ----"
+export path_hwtools=$BUILD_SOURCESDIRECTORY/hwtools/head/setup
+export PATH=$path_hwtools:$PATH
+
+echo "---- Run hwsetup ----"
+# This script sets the XILINX_VIVADO, MODELSIM, and LIB_BASE_PATH
+# variables based on the agent's configuration so we can find the EDA
+# tools.
+pushd $BUILD_SOURCESDIRECTORY/uhddev/fpga/.ci/hwtools
+source hwsetup.sh
+popd
+
+echo "---- Install Vivado patches ----"
+pushd $BUILD_SOURCESDIRECTORY
+wget -q $PATCHES_PATH/2019.1/AR73068_Vivado_2019_1_preliminary_rev1.zip -O ./AR73068_Vivado_2019_1_preliminary_rev1.zip
+unzip -q -o AR73068_Vivado_2019_1_preliminary_rev1.zip -d ./patch/
+export XILINX_PATH=$PWD/patch/vivado
+popd
+
+echo "---- Run setupenv ----"
+export MSIM_VIV_COMPLIBDIR=$LIB_BASE_PATH/vivado/2019.1.1/modelsim_SE-64_2020
+source $1/setupenv.sh --vivado-path $(dirname $XILINX_VIVADO) --modelsim-path $(dirname $MODELSIM)
diff --git a/fpga/.ci/templates/check_clean_repo_steps.yml b/fpga/.ci/templates/check_clean_repo_steps.yml
index 7ac034dd8..08eb670ba 100644
--- a/fpga/.ci/templates/check_clean_repo_steps.yml
+++ b/fpga/.ci/templates/check_clean_repo_steps.yml
@@ -12,7 +12,7 @@ parameters:
   # Directory to execute the commands in
   - name: directory
     type: string
-    default: $(Agent.BuildDirectory)/s
+    default: $(Agent.BuildDirectory)/s/uhddev
 
 steps:
 # Windows based calls
diff --git a/fpga/.ci/templates/fpga_build.yml b/fpga/.ci/templates/fpga_build.yml
deleted file mode 100644
index dbc41b790..000000000
--- a/fpga/.ci/templates/fpga_build.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Copyright 2021 Ettus Research, a National Instruments Brand
-#
-# SPDX-License-Identifier: LGPL-3.0-or-later
-#
-# Description:
-#
-#   This template should be used for all FPGA builds within the uhddev
-#   repository. Exports the FPGA build results (bitfile, device tree and timing
-#   report from build directory) as artifacts 'FPGA image <target>'.
-#
-#   See description for the parameters below.
-#
-
-parameters:
-  ### Required parameters
-  # FPGA target to be built e.g. X410_XG
-  - name: target
-    type: string
-
-  ### Optional parameters
-  # Set to true if the intermediate files from build-<target> directory should
-  # be exported for debugging as artifact 'FPGA build <target> (<attempt>)'
-  - name: debug
-    type: boolean
-    default: false
-  # Checkout repository in a clean state as described in
-  # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema%2Cparameter-schema#checkout
-  - name: clean
-    type: boolean
-    default: true
-  # Default timeout of 4h
-  - name: timeout
-    type: number
-    default: 240
-
-jobs:
-- job: FPGA_${{ parameters.target }}
-  displayName: 'Build FPGA ${{ parameters.target }}'
-  pool:
-    name: de-dre-lab
-    demands:
-    - ettus_fpga_build
-    - vivado2019.1
-  timeoutInMinutes: ${{ parameters.timeout }}
-  steps:
-    # Currently limited to be executed in same repo.
-    # Removes all unversioned files if necessary.
-    # Checkout path defined by single repository case in
-    # https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops#checkout-path
-    - checkout: self
-      clean: ${{ parameters.clean }}
-
-    # Remove incomplete IP builds due to aborted previous runs.
-    - bash: |
-        python3 cleanup_incomplete_ip_builds.py -d fpga/usrp3/top/x400
-      workingDirectory: $(Agent.BuildDirectory)/s/fpga/.ci/scripts/
-      displayName: 'Incomplete IP Cleanup'
-
-    # Clean export directories and run FPGA build.
-    - bash: |
-        rm -rf build
-        rm -rf build-${{ parameters.target }}
-        source setupenv.sh
-        make ${{ parameters.target }}
-      workingDirectory: $(Agent.BuildDirectory)/s/fpga/usrp3/top/x400
-      displayName: 'Build Target'
-
-    # Publish the final result only if all previous steps passed
-    - publish: $(Agent.BuildDirectory)/s/fpga/usrp3/top/x400/build
-      artifact: 'FPGA image ${{ parameters.target }}'
-      displayName: 'Publish FPGA'
-
-    # Publish intermediate files.
-    - publish: $(Agent.BuildDirectory)/s/fpga/usrp3/top/x400/build-${{ parameters.target }}
-      artifact: 'FPGA build ${{ parameters.target }} ($(System.JobAttempt))'
-      condition: and(always(), eq('${{ parameters.debug }}', true))
-      displayName: 'Publish Build Directory'
-
-    # Check if FPGA build left any untracked files.
-    - ${{ if eq(parameters.debug, true) }}:
-      - template: check_clean_repo_steps.yml
diff --git a/fpga/.ci/templates/job-build-fpga.yml b/fpga/.ci/templates/job-build-fpga.yml
new file mode 100644
index 000000000..eb8a1f5aa
--- /dev/null
+++ b/fpga/.ci/templates/job-build-fpga.yml
@@ -0,0 +1,94 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This template shall be used for FPGA builds within the uhddev
+#   repository. Exports the FPGA build results (bitfile, device tree and build
+#   report from build directory) as artifacts '<target>'.
+#
+#   See description for the parameters below.
+#
+
+parameters:
+### Required parameters
+# FPGA targets to be built.
+# E.g.
+#   targets_matrix:
+#     X410_XG_100:
+#       target: X410_XG_100
+#       timeout: 480
+#     X410_X4_200:
+#       target: X410_X4_200
+#       timeout: 480
+- name: targets_matrix
+  type: object
+# IP artifact name (from pipeline) to be used
+- name: ip_artifact
+  type: string
+# PATH to device's top directory (where Makefile is located)
+- name: top_dir
+  type: string
+
+### Optional parameters
+# Set to true if the intermediate files from build-<target> directory should
+# be exported for debugging as artifact 'Build artifacts <target> (Attempt <attempt>)'
+- name: publish_int_files
+  type: boolean
+  default: false
+
+
+jobs:
+- job: build_fpga
+  displayName: 'Build FPGA'
+  timeoutInMinutes: $[ variables['timeout'] ]
+  pool:
+    name: hardware
+  variables:
+  - group: sdr-pipeline-vars
+  strategy:
+    matrix:
+      ${{ parameters.targets_matrix }}
+  steps:
+    - checkout: self
+      clean: true
+
+    - checkout: hwtools
+      clean: true
+      path: s/hwtools/head
+      persistCredentials: true
+
+    - download: current
+      artifact: ${{ parameters.ip_artifact }}
+      displayName: 'Download ${{ parameters.ip_artifact }}'
+
+    - bash: |
+        mv $(Pipeline.Workspace)/${{ parameters.ip_artifact }} \
+           ${{ parameters.top_dir }}/build-ip/
+      displayName: 'Populate build-ip'
+
+    - bash: |
+        source $(Build.SourcesDirectory)/uhddev/fpga/.ci/scripts/run_setup.sh ./
+        make $(target_name)
+      workingDirectory: ${{ parameters.top_dir }}
+      env:
+        PATCHES_PATH: $(sdr-vivado-patches)
+      displayName: 'Build Target'
+
+    # Publish the final result only if all previous steps passed
+    - publish: ${{ parameters.top_dir }}/build
+      artifact: '$(target_name)'
+      displayName: 'Publish FPGA'
+
+    # Publish intermediate files.
+    - publish: ${{ parameters.top_dir }}/build-$(target_name)
+      artifact: 'Build artifacts $(target_name) (Attempt $(System.JobAttempt))'
+      condition: and(always(), eq('${{ parameters.publish_int_files }}', true))
+      displayName: 'Publish Build Directory'
+
+    # Check if FPGA build left any untracked files.
+    - template: check_clean_repo_steps.yml
+      parameters:
+        directory: ${{ parameters.top_dir }}
diff --git a/fpga/.ci/templates/job-build-ip.yml b/fpga/.ci/templates/job-build-ip.yml
new file mode 100644
index 000000000..ab58b9fcc
--- /dev/null
+++ b/fpga/.ci/templates/job-build-ip.yml
@@ -0,0 +1,98 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This template should be used for IP builds within the uhddev repository.
+#   This job uses Azure's caching mechanism to speed up the build. A
+#   successful job publishes the corresponding build-ip/ directory as a
+#   pipeline artifact.
+#
+#   See description for the parameters below.
+#
+
+parameters:
+### Required parameters
+# PATH to device's top directory (where Makefile is located)
+- name: directory
+  type: string
+# Name of the IP Make target (e.g. X410_IP for X410's IP)
+- name: ip_target
+  type: string
+
+### Optional parameters
+# Option to ignore cached artifacts (when available) and perform
+# a clean IP build.
+- name: clean_build
+  type: boolean
+  default: false
+
+jobs:
+- job: ${{ parameters.ip_target }}
+  displayName: 'Build ${{ parameters.ip_target }}'
+  pool:
+    name: Hardware
+  timeoutInMinutes: 240
+  variables:
+  - group: sdr-pipeline-vars
+  steps:
+    - checkout: self
+      clean: true
+      persistCredentials: true
+
+    - checkout: hwtools
+      clean: true
+      path: s/hwtools/head
+      persistCredentials: true
+
+    # Pipeline caching is leveraged in this job to minimize the time it takes
+    # to build all the IP.
+    # At a high level, the pipeline checks for an available and valid cache
+    # artifact from its scope (target branch in PR or its own branch) and
+    # downloads it to the workspace (Cache hit).
+    # If no valid cache artifact exists (Cache miss), then the build will run
+    # and the pipeline will automatically save the result as a cache artifact
+    # for subsequent runs.
+    # Further details on "Pipeline caching" available online:
+    #   https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops
+    - ${{ if eq(parameters.clean_build, false) }}:
+      - task: Cache@2
+        inputs:
+          key: "uhddev-fpga-${{ parameters.ip_target }}"
+          path: ${{ parameters.directory }}/build-ip
+          cacheHitVar: CACHE_RESTORED
+        displayName: Cache IP
+
+    - bash: |
+        BUILD_IP_CACHE_HASH=`cat build-ip/.ci/build-hash`
+        echo "BUILD_IP_CACHE_HASH=$BUILD_IP_CACHE_HASH"
+        git fetch origin $BUILD_IP_CACHE_HASH
+        bash $(Build.SourcesDirectory)/uhddev/fpga/.ci/scripts/refresh_ip.sh \
+          `realpath build-ip/` $BUILD_IP_CACHE_HASH
+      workingDirectory: ${{ parameters.directory }}
+      condition: eq(variables.CACHE_RESTORED, 'true')
+      displayName: "Refresh IP"
+
+    - bash: |
+        source $(Build.SourcesDirectory)/uhddev/fpga/.ci/scripts/run_setup.sh ./
+        make ${{ parameters.ip_target }}
+        mkdir -p build-ip/.ci/
+        git rev-parse --verify HEAD 2>/dev/null 1>build-ip/.ci/build-hash
+      workingDirectory: ${{ parameters.directory }}
+      env:
+        PATCHES_PATH: $(sdr-vivado-patches)
+      displayName: "Build IP"
+
+    - publish: ${{ parameters.directory }}/build-ip
+      artifact: '${{ parameters.ip_target }} (Attempt $(System.JobAttempt))'
+      condition: failed()
+
+    - publish: ${{ parameters.directory }}/build-ip
+      artifact: ${{ parameters.ip_target }}
+      condition: succeeded()
+
+    - template: check_clean_repo_steps.yml
+      parameters:
+        directory: ${{ parameters.directory }}
diff --git a/fpga/.ci/templates/job-package-images.yml b/fpga/.ci/templates/job-package-images.yml
new file mode 100644
index 000000000..df7350042
--- /dev/null
+++ b/fpga/.ci/templates/job-package-images.yml
@@ -0,0 +1,71 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This template shall be used to package images.
+#
+#   See description for the parameters below.
+#
+
+parameters:
+### Required parameters
+# Package name for identification purposes
+- name: package_name
+  type: string
+# FPGA artifacts to be downloaded to "build_directory"
+# Note that the value for each key is not used in this job.
+# E.g.
+#   targets_matrix:
+#     X410_XG_100:
+#       target: X410_XG_100
+#       timeout: 480
+#     X410_X4_200:
+#       target: X410_X4_200
+#       timeout: 480
+- name: artifacts_matrix
+  type: object
+# PATH to device's build directory (where artifacts will be copied and
+# packaging utility will be run)
+- name: build_directory
+  type: string
+
+
+jobs:
+- job: ${{ parameters.package_name }}
+  displayName: 'Create ${{ parameters.package_name }}'
+  timeoutInMinutes: 10
+  pool:
+    name: hardware
+  steps:
+    - checkout: self
+      clean: true
+
+    - ${{ each artifact in parameters.artifacts_matrix }}:
+      - download: current
+        artifact: ${{ artifact.key }}
+        displayName: 'Download ${{ artifact.key }}'
+
+    - ${{ each artifact in parameters.artifacts_matrix }}:
+      - bash: |
+          mkdir -p ${{ parameters.build_directory }}
+          cp $(Pipeline.Workspace)/${{ artifact.key }}/* \
+             ${{ parameters.build_directory }}/
+          rm -rf $(Pipeline.Workspace)/${{ artifact.key }}/
+        displayName: 'Populate ${{ artifact.key }} artifacts'
+
+    - bash: |
+        OSS_REPO_HASH="$(git rev-parse --verify HEAD --short=7 2>/dev/null)"
+        echo "##vso[task.setvariable variable=OSS_REPO_HASH]$OSS_REPO_HASH"
+        echo "OSS Repo hash: $OSS_REPO_HASH"
+        python3 $(Build.SourcesDirectory)/fpga/usrp3/tools/utils/package_images.py --githash "uhd-$OSS_REPO_HASH"
+        rm -f *.bit *.dts *.rpt *.md5
+      workingDirectory: ${{ parameters.build_directory }}
+      displayName: 'Run package_images.py'
+
+    # Publish the final result only if all previous steps passed
+    - publish: ${{ parameters.build_directory }}
+      artifact: '${{ parameters.package_name }}-g$(OSS_REPO_HASH)'
+      displayName: 'Publish package'
diff --git a/fpga/.ci/templates/job-run-testbenches.yml b/fpga/.ci/templates/job-run-testbenches.yml
new file mode 100644
index 000000000..6f6ef142b
--- /dev/null
+++ b/fpga/.ci/templates/job-run-testbenches.yml
@@ -0,0 +1,40 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This template is used to run all the testbenches using ModelSim.
+#
+
+
+jobs:
+- job:
+  displayName: "ModelSim Simulation"
+  timeoutInMinutes: 360
+  pool:
+    name: Hardware
+  variables:
+  - group: sdr-pipeline-vars
+  steps:
+    - checkout: self
+      clean: true
+      persistCredentials: true
+
+    - checkout: hwtools
+      clean: true
+      path: s/hwtools/head
+      persistCredentials: true
+
+    - bash: |
+        source $(Build.SourcesDirectory)/uhddev/fpga/.ci/scripts/run_setup.sh ./top/x300
+
+        echo "---- Run testbenches ----"
+        pushd ./tools/utils
+        python3 ./run_testbenches.py --logged --simulator modelsim --excludes=modelsim.excludes -j2 run
+        popd
+      workingDirectory: uhddev/fpga/usrp3
+      env:
+        PATCHES_PATH: $(sdr-vivado-patches)
+      displayName: "Run Testbenches"
diff --git a/fpga/.ci/templates/stages-fpga-pipeline.yml b/fpga/.ci/templates/stages-fpga-pipeline.yml
new file mode 100644
index 000000000..3bd98e390
--- /dev/null
+++ b/fpga/.ci/templates/stages-fpga-pipeline.yml
@@ -0,0 +1,104 @@
+#
+# Copyright 2022 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Description:
+#
+#   This template defines a stages-based pipeline that may be leveraged by both
+#   a PR pipeline or a continuous delivery pipeline.
+#
+#   See https://aka.ms/yaml for pipeline YAML documentation.
+#
+
+parameters:
+## Optional parameters
+# Run testbenches
+- name: run_testbenches
+  type: boolean
+  default: true
+# Option to ignore cached artifacts (if available) and perform
+# a clean IP build.
+- name: clean_ip_build
+  type: boolean
+  default: false
+# Build X410 FPGA targets
+- name: build_x410
+  type: boolean
+  default: true
+# X410 FPGA targets to build (if build_x410 is true)
+- name: x410_targets_matrix
+  type: object
+  default:
+    X410_X4_200:
+      target_name: X410_X4_200
+      timeout: 480
+# Option to publish intermediate files
+- name: publish_int_files
+  type: boolean
+  default: false
+# Create images package
+- name: package_images
+  type: boolean
+  default: false
+
+
+resources:
+  repositories:
+  - repository: hwtools
+    type: git
+    ref: main
+    name: DevCentral/hwtools
+
+
+stages:
+
+### START: Testbenches stage
+
+- stage: run_testbenches_stage
+  displayName: Run Testbenches
+  dependsOn: []
+  condition: and(succeeded(), eq('${{ parameters.run_testbenches }}', 'true'))
+  jobs:
+  - template: job-run-testbenches.yml
+
+### END: Testbenches stage
+
+
+### START: X410 stages
+
+- stage: build_x410_ip_stage
+  displayName: Build X410 IP
+  dependsOn: []
+  condition: and(succeeded(), eq('${{ parameters.build_x410 }}', 'true'))
+  jobs:
+  - template: job-build-ip.yml
+    parameters:
+      directory: uhddev/fpga/usrp3/top/x400
+      ip_target: X410_IP
+      clean_build: ${{ parameters.clean_ip_build }}
+
+- stage: build_x410_targets_stage
+  displayName: Build X410 FPGA Targets
+  dependsOn: build_x410_ip_stage
+  condition: and(succeeded('build_x410_ip_stage'), eq('${{ parameters.build_x410 }}', 'true'))
+  jobs:
+  - template: job-build-fpga.yml
+    parameters:
+      targets_matrix: ${{ parameters.x410_targets_matrix }}
+      ip_artifact: X410_IP
+      top_dir: uhddev/fpga/usrp3/top/x400
+      publish_int_files: ${{ parameters.publish_int_files }}
+
+- stage: create_x410_packages_stage
+  displayName: Create X410 Packages
+  dependsOn: build_x410_targets_stage
+  condition: and(succeeded('build_x410_targets_stage'), eq('${{ parameters.package_images }}', 'true'))
+  jobs:
+  - template: job-package-images.yml
+    parameters:
+      package_name: x4xx_x410_fpga_default
+      artifacts_matrix: ${{ parameters.x410_targets_matrix }}
+      build_directory: uhddev/fpga/usrp3/top/x400/build/
+
+### END: X410 stages
diff --git a/fpga/.ci/x4xx-pr-check.yml b/fpga/.ci/x4xx-pr-check.yml
deleted file mode 100644
index c1a26edf9..000000000
--- a/fpga/.ci/x4xx-pr-check.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-#
-# Copyright 2021 Ettus Research, a National Instruments Brand
-#
-# SPDX-License-Identifier: LGPL-3.0-or-later
-#
-# Description:
-#
-#   This pipeline is used to test building  the FPGA and CPLD for each pull
-#   request.
-#
-#   See https://aka.ms/yaml for pipeline YAML documentation.
-#
-
-trigger:
-- none
-
-# Filter for target branches and paths on PRs. See:
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml#pr-triggers
-pr:
-  branches:
-    include:
-    - master
-  paths:
-    include:
-    - fpga/usrp3/lib
-    - fpga/usrp3/tools
-    - fpga/usrp3/top/x400
-    - fpga/.ci
-
-jobs:
-# -------------------------------------------------------------------
-# Build XG FPGA (100 MHz)
-# -------------------------------------------------------------------
-- template: templates/fpga_build.yml
-  parameters:
-    target: X410_XG_100
-    debug: true  # to be able to debug any failed attempts
-    clean: false # for speedup of PR testing
-    timeout: 480
-
-# -------------------------------------------------------------------
-# Build X4 FPGA (200 MHz)
-# -------------------------------------------------------------------
-- template: templates/fpga_build.yml
-  parameters:
-    target: X410_X4_200
-    debug: true  # to be able to debug any failed attempts
-    clean: false # for speedup of PR testing
-    timeout: 480
-
-# -------------------------------------------------------------------
-# Build C1 FPGA (400 MHz)
-# -------------------------------------------------------------------
-- template: templates/fpga_build.yml
-  parameters:
-    target: X410_C1_400
-    debug: true  # to be able to debug any failed attempts
-    clean: false # for speedup of PR testing
-    timeout: 480
-
-# -------------------------------------------------------------------
-# Make CPLD
-# -------------------------------------------------------------------
-#- template: templates/mb_cpld_build.yml
-#  parameters:
-#    debug: true  # to be able to debug any failed attempts
-
-# -------------------------------------------------------------------
-# Make ZBX CPLD
-# -------------------------------------------------------------------
-#- template: templates/zbx_cpld_build.yml
-#  parameters:
-#    debug: true  # to be able to debug any failed attempts
-
-# -------------------------------------------------------------------
-# Build IP
-# -------------------------------------------------------------------
-- job: IP
-  displayName: 'Build IP'
-  pool:
-    name: de-dre-lab
-    demands:
-    - ettus_fpga_build
-    - vivado2019.1
-  timeoutInMinutes: 120
-  steps:
-    - checkout: self
-      clean: false
-
-    - bash: |
-        python3 cleanup_incomplete_ip_builds.py -d $BUILD_SOURCESDIRECTORY/fpga/usrp3/top/x400
-      workingDirectory: fpga/.ci/scripts/
-      displayName: 'Incomplete IP Cleanup'
-
-    # Delete everything except for the build-ip directory, which is expected to
-    # remain for speed-up.
-    - bash: |
-        git clean -xdff -e build-ip
-      displayName: 'git clean'
-
-    - bash: |
-        source setupenv.sh
-        make X410_IP
-      workingDirectory: fpga/usrp3/top/x400
-
-    - publish: fpga/usrp3/top/x400/build-ip
-      artifact: 'IP ($(System.JobAttempt))'
-      condition: failed()
-
-    - publish: fpga/usrp3/top/x400/build-ip
-      artifact: 'IP'
-      condition: succeeded()
-
-    - template: templates/check_clean_repo_steps.yml