n3xx: enable DRP access to DB MGTS & QPLLs

 - re-structured Magnesium.init_jesd() sequence to accomodate the reconfiguration
 - added DRP access protocols to nijesdcore.py
 - cleaned up a few comments and log statements in nijesdcore.py for flow

Reviewed-By: Martin Braun <martin.braun@ettus.com>

2 files changed, 249 insertions, 36 deletions

diff --git a/mpm/python/usrp_mpm/dboard_manager/magnesium.py b/mpm/python/usrp_mpm/dboard_manager/magnesium.py
index fdcaea0de..9cbb2bd9b 100644
--- a/mpm/python/usrp_mpm/dboard_manager/magnesium.py
+++ b/mpm/python/usrp_mpm/dboard_manager/magnesium.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import os
 import time
 import threading
+import math
 from six import iterkeys, iteritems
 from . import lib # Pulls in everything from C++-land
 from .base import DboardManagerBase
@@ -316,6 +317,7 @@ class Magnesium(DboardManagerBase):
     # point, so we set it to the (carefully calculated) alternate value instead.
     INIT_PHASE_DAC_WORD = 31000 # Intentionally decimal
     default_master_clock_rate = 125e6
+    default_current_jesd_rate = 2500e6
 
     def __init__(self, slot_idx, **kwargs):
         super(Magnesium, self).__init__(slot_idx, **kwargs)
@@ -327,8 +329,9 @@ class Magnesium(DboardManagerBase):
         # This is a default ref clock freq, it must be updated before init() is
         # called!
         self.ref_clock_freq = 10e6
-        # This will get updated during init()
+        # These will get updated during init()
         self.master_clock_rate = None
+        self.current_jesd_rate = None
         # Predeclare some attributes to make linter happy:
         self.lmk = None
         self._port_expander = None
@@ -567,19 +570,6 @@ class Magnesium(DboardManagerBase):
         return True
 
 
-    def cpld_peek(self, addr):
-        """
-        Debug for accessing the CPLD via the RPC shell.
-        """
-        return self.cpld.peek16(addr)
-
-    def cpld_poke(self, addr, data):
-        """
-        Debug for accessing the CPLD via the RPC shell.
-        """
-        self.cpld.poke16(addr, data)
-        return self.cpld.peek16(addr)
-
     def init_rf_cal(self, args):
         " Setup RF CAL "
         self.log.info("Setting up RF CAL...")
@@ -621,7 +611,17 @@ class Magnesium(DboardManagerBase):
         All clocks must be set up and stable before starting this routine.
         """
         jesdcore.check_core()
-        jesdcore.init()
+
+        # JESD Lane Rate only depends on the master_clock_rate selection, since all
+        # other link parameters (LMFS,N) remain constant.
+        L = 4
+        M = 4
+        F = 2
+        S = 1
+        N = 16
+        new_rate = self.master_clock_rate * M * N * (10.0/8) / L / S
+        self.log.trace("Calculated JESD204b lane rate is {} Gbps".format(new_rate/1e9))
+        self.set_jesd_rate(jesdcore, new_rate)
 
         self.log.trace("Pulsing Mykonos Hard Reset...")
         self.cpld.reset_mykonos()
@@ -639,7 +639,7 @@ class Magnesium(DboardManagerBase):
         self.log.trace("Starting JESD204b Link Initialization...")
         # Generally, enable the source before the sink. Start with the DAC side.
         self.log.trace("Starting FPGA framer...")
-        jesdcore.init_framer()
+        jesdcore.init_framer(bypass_scrambler=True)
         self.log.trace("Starting Mykonos deframer...")
         self.mykonos.start_jesd_rx()
         # Now for the ADC link. Note that the Mykonos framer will not start issuing CGS
@@ -648,7 +648,6 @@ class Magnesium(DboardManagerBase):
         # start the deframer in the FPGA.
         self.log.trace("Starting Mykonos framer...")
         self.mykonos.start_jesd_tx()
-        self.log.trace("Enable FPGA SYSREF Receiver.")
         jesdcore.enable_lmfc(True)
         jesdcore.send_sysref_pulse()
         self.log.trace("Starting FPGA deframer...")
@@ -674,14 +673,109 @@ class Magnesium(DboardManagerBase):
         self.log.info("JESD204B Link Initialization & Training Complete")
 
 
-    def dump_jesd_core(self):
-        " Debug method to dump all JESD core regs "
-        dboard_ctrl_regs = UIO(label="dboard-regs-{}".format(self.slot_idx))
-        for i in range(0x2000, 0x2110, 0x10):
-            print(("0x%04X " % i), end=' ')
-            for j in range(0, 0x10, 0x4):
-                print(("%08X" % dboard_ctrl_regs.peek32(i + j)), end=' ')
-            print("")
+    def set_jesd_rate(self, jesdcore, new_rate, force=False):
+        """
+        Make the QPLL and GTX changes required to change the JESD204B core rate.
+        """
+        # The core is directly compiled for 125 MHz sample rate, which
+        # corresponds to a lane rate of 2.5 Gbps. The same QPLL and GTX settings apply
+        # for the 122.88 MHz sample rate.
+        #
+        # The higher LTE rate, 153.6 MHz, requires changes to the default configuration
+        # of the MGT components. This function performs the required changes in the
+        # following order (as recommended by UG476).
+        #
+        # 1) Modify any QPLL settings.
+        # 2) Perform the QPLL reset routine by pulsing reset then waiting for lock.
+        # 3) Modify any GTX settings.
+        # 4) Perform the GTX reset routine by pulsing reset and waiting for reset done.
+
+        assert new_rate in (2457.6e6, 2500e6, 3072e6)
+
+        # On first run, we have no idea how the FPGA is configured... so let's force an
+        # update to our rate.
+        force = force or (self.current_jesd_rate is None)
+
+        skip_drp = False
+        if not force:
+            #           Current     New       Skip?
+            skip_drp = {2457.6e6 : {2457.6e6: True,  2500.0e6: True,  3072.0e6:False},
+                        2500.0e6 : {2457.6e6: True,  2500.0e6: True,  3072.0e6:False},
+                        3072.0e6 : {2457.6e6: False, 2500.0e6: False, 3072.0e6:True}}[self.current_jesd_rate][new_rate]
+
+        if skip_drp:
+            self.log.trace("Current lane rate is compatible with the new rate. Skipping "
+                           "reconfiguration.")
+
+        # These are the only registers in the QPLL and GTX that change based on the
+        # selected line rate. The MGT wizard IP was generated for each of the rates and
+        # reference clock frequencies and then diffed to create this table.
+        QPLL_CFG         = {2457.6e6: 0x680181, 2500e6: 0x680181, 3072e6: 0x06801C1}[new_rate]
+        QPLL_FBDIV       = {2457.6e6:    0x120, 2500e6:    0x120, 3072e6:      0x80}[new_rate]
+        MGT_PMA_RSV      = {2457.6e6: 0x1E7080, 2500e6: 0x1E7080, 3072e6:   0x18480}[new_rate]
+        MGT_RX_CLK25_DIV = {2457.6e6:        5, 2500e6:        5, 3072e6:         7}[new_rate]
+        MGT_TX_CLK25_DIV = {2457.6e6:        5, 2500e6:        5, 3072e6:         7}[new_rate]
+        MGT_RXOUT_DIV    = {2457.6e6:        4, 2500e6:        4, 3072e6:         2}[new_rate]
+        MGT_TXOUT_DIV    = {2457.6e6:        4, 2500e6:        4, 3072e6:         2}[new_rate]
+        MGT_RXCDR_CFG    = {2457.6e6:0x03000023ff10100020, 2500e6:0x03000023ff10100020, 3072e6:0x03000023ff10200020}[new_rate]
+
+
+        # 1-2) Do the QPLL first
+        if not skip_drp:
+            self.log.trace("Changing QPLL settings to support {} Gbps".format(new_rate/1e9))
+            jesdcore.set_drp_target('qpll', 0)
+            # QPLL_CONFIG is spread across two regs: 0x32 (dedicated) and 0x33 (shared)
+            reg_x32 = QPLL_CFG & 0xFFFF # [16:0] -> [16:0]
+            reg_x33 = jesdcore.drp_access(rd = True, addr = 0x33)
+            reg_x33 = (reg_x33 & 0xF800) | ((QPLL_CFG >> 16) & 0x7FF)  # [26:16] -> [11:0]
+            jesdcore.drp_access(rd = False, addr = 0x32, wr_data = reg_x32)
+            jesdcore.drp_access(rd = False, addr = 0x33, wr_data = reg_x33)
+            # QPLL_FBDIV is shared with other settings in reg 0x36
+            reg_x36 = jesdcore.drp_access(rd = True, addr = 0x36)
+            reg_x36 = (reg_x36 & 0xFC00) | (QPLL_FBDIV & 0x3FF)  # in bits [9:0]
+            jesdcore.drp_access(rd = False, addr = 0x36, wr_data = reg_x36)
+
+        # Run the QPLL reset sequence and prep the MGTs for modification.
+        jesdcore.init()
+
+        # 3-4) And the 4 MGTs second
+        if not skip_drp:
+            self.log.trace("Changing MGT settings to support {} Gbps".format(new_rate/1e9))
+            for lane in range(0,4):
+                jesdcore.set_drp_target('mgt', lane)
+                # MGT_PMA_RSV is split over 0x99 (LSBs) and 0x9A
+                reg_x99 = MGT_PMA_RSV & 0xFFFF
+                reg_x9A = (MGT_PMA_RSV >> 16) & 0xFFFF
+                jesdcore.drp_access(rd = False, addr = 0x99, wr_data = reg_x99)
+                jesdcore.drp_access(rd = False, addr = 0x9A, wr_data = reg_x9A)
+                # MGT_RX_CLK25_DIV is embedded with others in 0x11. The encoding for
+                # the DRP register value is one less than the desired value.
+                reg_x11 = jesdcore.drp_access(rd = True, addr = 0x11)
+                reg_x11 = (reg_x11 & 0xF83F) | ((MGT_RX_CLK25_DIV-1 & 0x1F) << 6) # [10:6]
+                jesdcore.drp_access(rd = False, addr = 0x11, wr_data = reg_x11)
+                # MGT_TX_CLK25_DIV is embedded with others in 0x6A. The encoding for
+                # the DRP register value is one less than the desired value.
+                reg_x6A = jesdcore.drp_access(rd = True, addr = 0x6A)
+                reg_x6A = (reg_x6A & 0xFFE0) | (MGT_TX_CLK25_DIV-1 & 0x1F) # [4:0]
+                jesdcore.drp_access(rd = False, addr = 0x6A, wr_data = reg_x6A)
+                # MGT_RXCDR_CFG is split over 0xA8 (LSBs) through 0xAD
+                RXCDR_REG_BASE = 0xA8
+                for reg_num in range(0, 6):
+                    reg_addr = RXCDR_REG_BASE + reg_num
+                    reg_data = (MGT_RXCDR_CFG >> 16*reg_num) & 0xFFFF
+                    jesdcore.drp_access(rd = False, addr = reg_addr, wr_data = reg_data)
+                # MGT_RXOUT_DIV and MGT_TXOUT_DIV are embedded together in 0x88. The
+                # encoding for the DRP register value is drp_val=log2(attribute)
+                reg_x88 = (int(math.log(MGT_RXOUT_DIV,2)) & 0x7) | \
+                         ((int(math.log(MGT_TXOUT_DIV,2)) & 0x7) << 4) # RX=[2:0] TX=[6:4]
+                jesdcore.drp_access(rd = False, addr = 0x88, wr_data = reg_x88)
+            self.log.trace("GTX settings changed to support {} Gbps".format(new_rate/1e9))
+            jesdcore.disable_drp_target()
+
+        self.log.trace("JESD204b Lane Rate set to {} Gbps!".format(new_rate/1e9))
+        self.current_jesd_rate = new_rate
+        return
+
 
     def get_user_eeprom_data(self):
         """
@@ -837,3 +931,57 @@ class Magnesium(DboardManagerBase):
             'value': str(lock_status).lower(),
         }
 
+
+    ##########################################################################
+    # Debug
+    ##########################################################################
+    def cpld_peek(self, addr):
+        """
+        Debug for accessing the CPLD via the RPC shell.
+        """
+        return self.cpld.peek16(addr)
+
+    def cpld_poke(self, addr, data):
+        """
+        Debug for accessing the CPLD via the RPC shell.
+        """
+        self.cpld.poke16(addr, data)
+        return self.cpld.peek16(addr)
+
+    def dump_jesd_core(self):
+        " Debug method to dump all JESD core regs "
+        dboard_ctrl_regs = UIO(
+            label="dboard-regs-{}".format(self.slot_idx),
+            read_only=False
+        )
+        for i in range(0x2000, 0x2110, 0x10):
+            print(("0x%04X " % i), end=' ')
+            for j in range(0, 0x10, 0x4):
+                print(("%08X" % dboard_ctrl_regs.peek32(i + j)), end=' ')
+            print("")
+        dboard_ctrl_regs = None
+
+    def dbcore_peek(self, addr):
+        """
+        Debug for accessing the DB Core registers via the RPC shell.
+        """
+        dboard_ctrl_regs = UIO(
+            label="dboard-regs-{}".format(self.slot_idx),
+            read_only=False
+        )
+        rd_data = dboard_ctrl_regs.peek32(addr)
+        self.log.trace("DB Core Register 0x{:04X} response: 0x{:08X}".format(addr, rd_data))
+        dboard_ctrl_regs = None
+        return rd_data
+
+    def dbcore_poke(self, addr, data):
+        """
+        Debug for accessing the DB Core registers via the RPC shell.
+        """
+        dboard_ctrl_regs = UIO(
+            label="dboard-regs-{}".format(self.slot_idx),
+            read_only=False
+        )
+        self.log.trace("Writing DB Core Register 0x{:04X} with 0x{:08X}...".format(addr, data))
+        dboard_ctrl_regs.poke32(addr, data)
+        dboard_ctrl_regs = None
diff --git a/mpm/python/usrp_mpm/nijesdcore.py b/mpm/python/usrp_mpm/nijesdcore.py
index 00c557fdd..f46809d94 100644
--- a/mpm/python/usrp_mpm/nijesdcore.py
+++ b/mpm/python/usrp_mpm/nijesdcore.py
@@ -43,6 +43,7 @@ class NIMgJESDCore(object):
     MGT_TX_TRANSCEIVER_CONTROL = 0x2064
     MGT_TX_SCRAMBLER_CONTROL   = 0x2068
     LMK_SYNC_CONTROL           = 0x206C
+    JESD_MGT_DRP_CONTROL       = 0x2070
     SYSREF_CAPTURE_CONTROL     = 0x2078
     JESD_SIGNATURE_REG         = 0x2100
     JESD_REVISION_REG          = 0x2104
@@ -99,25 +100,28 @@ class NIMgJESDCore(object):
         self._gt_reset('rx', reset_only=False)
         self.regs.poke32(self.MGT_RECEIVER_CONTROL, 0x0)
 
-    def init_framer(self):
+    def init_framer(self, bypass_scrambler = True):
         " Initialize framer "
         self.log.trace("Initializing framer...")
         # Disable DAC Sync from requesting CGS & Stop Deframer
-        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, 0x2002)
+        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, (0b1 << 13) | (0b1 << 1))
         # Reset, unreset, and check the GTs
         self._gt_reset('tx', reset_only=False)
         # MGT phy control... enable TX Driver Swing
         self.regs.poke32(self.MGT_TX_TRANSCEIVER_CONTROL, 0xF0000)
         time.sleep(0.001)
-        # Bypass scrambler and disable char replacement
-        self.regs.poke32(self.MGT_TX_SCRAMBLER_CONTROL, 0x1)
+        # Bypass scrambler and char replacement. If the scrambler is bypassed,
+        # then the char replacement is also disabled.
+        reg_val = {True: 0x01, False: 0x10}[bypass_scrambler]
+        self.regs.poke32(self.MGT_TX_SCRAMBLER_CONTROL, reg_val)
         # Check for Framer in Idle state
         rb = self.regs.peek32(self.MGT_TRANSMITTER_CONTROL)
         if rb & 0x100 != 0x100:
             raise Exception('TX Framer is not idle after reset')
-        # Enable the framer and incoming DAC Sync
-        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, 0x1000)
-        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, 0x0001)
+        # Enable incoming DAC Sync
+        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, 0b1 << 12)
+        # Enable the framer
+        self.regs.poke32(self.MGT_TRANSMITTER_CONTROL, 0b1 <<  0)
 
     def get_framer_status(self):
         " Return True if framer is in good status "
@@ -159,6 +163,7 @@ class NIMgJESDCore(object):
         """
         Enable/disable LMFC generator in FPGA.
         """
+        self.log.trace("%s FPGA SYSREF Receiver..." % {True: 'Enabling', False: 'Disabling'}[enable])
         disable_bit = 0b1
         if enable:
            disable_bit = 0b0
@@ -186,6 +191,7 @@ class NIMgJESDCore(object):
             for _ in range(20):
                 rb = self.regs.peek32(mgt_reg)
                 if rb & 0xFFFF0000 == 0x000F0000:
+                    self.log.trace("%s MGT Reset Cleared!" % tx_or_rx.upper())
                     return True
                 time.sleep(0.001)
             raise Exception('Timeout in GT {trx} Reset (Readback: 0x{rb:X})'.format(
@@ -198,8 +204,7 @@ class NIMgJESDCore(object):
         " Power down unused CPLLs and QPLLs "
         assert qplls in range(4+1) # valid is 0 - 4
         assert cplls in range(8+1) # valid is 0 - 8
-        self.log.trace("Powering down unused CPLLs and QPLLs...")
-        self.log.trace("Using {} CPLLs and {} QPLLs!".format(cplls, qplls))
+        self.log.trace("Powering up {} CPLLs and {} QPLLs".format(cplls, qplls))
         reg_val = 0xFFFF000F
         reg_val_on = 0x0
         # Power down state is when the corresponding bit is set. For the PLLs we wish to
@@ -230,6 +235,7 @@ class NIMgJESDCore(object):
         # Reset QPLLs.
         reg_val = 0x1111 # by default assert all resets
         self.regs.poke32(self.MGT_QPLL_CONTROL, reg_val)
+        self.log.trace("Resetting QPLL(s)...")
 
         # Unreset the PLLs in use and check for lock.
         if not reset_only:
@@ -240,14 +246,14 @@ class NIMgJESDCore(object):
                  reg_val_on = reg_val_on | 0x1 << nibble*4
               reg_val = reg_val ^ reg_val_on
               self.regs.poke32(self.MGT_QPLL_CONTROL, reg_val)
+              self.log.trace("Clearing QPLL reset...")
 
               # Check for lock a short time later.
               time.sleep(0.010)
               # Clear all QPLL sticky bits
               self.regs.poke32(self.MGT_QPLL_CONTROL, 0b1 << 16)
-              rb = self.regs.peek32(self.MGT_QPLL_CONTROL)
-              self.log.trace("Reading QPLL status register: {:04X}".format(rb & 0xFFFF))
               # Check for lock on active quads only.
+              rb = self.regs.peek32(self.MGT_QPLL_CONTROL)
               rb_mask = 0x0
               locked_val = 0x0
               for nibble in range(qplls):
@@ -257,4 +263,63 @@ class NIMgJESDCore(object):
                  rb_mask    = rb_mask    | 0xF << nibble*4
               if (rb & rb_mask) != locked_val:
                   raise Exception("One or more GT QPLLs failed to lock!")
+              self.log.trace("QPLL(s) reporting locked!")
+
+    def set_drp_target(self, mgt_or_qpll, dev_num):
+        """
+        Sets up access to the specified MGT or QPLL. This must be called
+        prior to drp_access(). It may be called repeatedly to change DRP targets
+        without calling the disable function first.
+        """
+        MAX_MGTS = 4
+        MAX_QPLLs = 1
+        DRP_ENABLE_VAL = 0b1
+        assert mgt_or_qpll.lower() in ('mgt', 'qpll')
+
+        self.log.trace("Enabling DRP access to %s #%d...",mgt_or_qpll.upper(), dev_num)
+
+        # Enable access to the DRP ports and select the correct channel. Channels are
+        # one-hot encoded with the MGT ports in bit locations [0, (MAX_MGTS-1)] and the
+        # QPLL in [MAX_MGTS, MAX_MGTs+MAX_QPLLs-1].
+        drp_ch_sel = {'mgt': dev_num, 'qpll': dev_num + MAX_MGTS}[mgt_or_qpll.lower()]
+        assert drp_ch_sel in range(MAX_MGTS + MAX_QPLLs)
+        reg_val = (0b1 << drp_ch_sel) | (DRP_ENABLE_VAL << 16)
+        self.log.trace("Writing DRP Control Register (offset 0x{:04X}) with 0x{:08X}"
+            .format(self.JESD_MGT_DRP_CONTROL, reg_val))
+        self.regs.poke32(self.JESD_MGT_DRP_CONTROL, reg_val)
+
+    def disable_drp_target(self):
+        """
+        Tears down access to the DRP ports. This must be called after drp_access().
+        """
+        self.regs.poke32(self.JESD_MGT_DRP_CONTROL, 0x0)
+        self.log.trace("DRP accesses disabled!")
+
+    def drp_access(self, rd = True, addr = 0, wr_data = 0):
+        """
+        Provides register access to the DRP ports on the MGTs or QPLLs buried inside
+        the JESD204b logic. Reads will return the DRP data directly. Writes will return
+        zeros.
+        """
+        # Check the DRP port is not busy.
+        if (self.regs.peek32(self.JESD_MGT_DRP_CONTROL) & (0b1 << 20)) != 0:
+            self.log.error("MGT/QPLL DRP Port is reporting busy during an attempted access.")
+            raise Exception("MGT/QPLL DRP Port is reporting busy during an attempted access.")
+
+        # Access the DRP registers...
+        rd_data = 0x0
+        core_offset = 0x2800 + (addr << 2)
+        if rd:
+            rd_data = self.regs.peek32(core_offset)
+            rd_data_valid = rd_data & 0xFFFF
+            self.log.trace("Reading DRP register 0x{:04X} at DB Core offset 0x{:04X}... "
+                           "0x{:04X}"
+                           .format(addr, core_offset, rd_data))
+        else:
+            self.log.trace("Writing DRP register 0x{:04X} with 0x{:04X}...".format(addr, wr_data))
+            self.regs.poke32(core_offset, wr_data)
+            if self.regs.peek32(core_offset) != wr_data:
+                self.log.error("DRP read after write failed to match!")
+
+        return rd_data