diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/dpd/Adapt.py | 291 | ||||
| -rw-r--r-- | python/dpd/ExtractStatistic.py | 37 | ||||
| -rw-r--r-- | python/dpd/GlobalConfig.py | 11 | ||||
| -rw-r--r-- | python/dpd/Measure.py | 36 | ||||
| -rw-r--r-- | python/dpd/Model_AM.py | 122 | ||||
| -rw-r--r-- | python/dpd/Model_PM.py | 124 | ||||
| -rw-r--r-- | python/dpd/Model_Poly.py | 147 | ||||
| -rw-r--r-- | python/dpd/RX_Agc.py | 39 | ||||
| -rwxr-xr-x | python/dpdce.py | 304 | ||||
| -rw-r--r-- | python/gui-dpdce.ini | 5 | ||||
| -rwxr-xr-x | python/gui.py | 6 | ||||
| -rwxr-xr-x | python/gui/api.py | 47 | ||||
| -rw-r--r-- | python/gui/static/css/odr.css | 14 | ||||
| -rw-r--r-- | python/gui/static/js/odr-home.js | 240 | ||||
| -rw-r--r-- | python/gui/static/js/odr-predistortion.js | 129 | ||||
| -rw-r--r-- | python/gui/static/js/odr-rcvalues.js | 29 | ||||
| -rw-r--r-- | python/gui/templates/about.html | 42 | ||||
| -rw-r--r-- | python/gui/templates/home.html | 72 | ||||
| -rw-r--r-- | python/gui/templates/modulator.html | 40 | ||||
| -rw-r--r-- | python/gui/templates/predistortion.html | 129 | ||||
| -rw-r--r-- | python/gui/templates/rcvalues.html | 39 | ||||
| -rw-r--r-- | python/lib/yamlrpc.py | 9 | ||||
| -rw-r--r-- | python/lib/zmqrc.py | 11 | 
23 files changed, 1180 insertions, 743 deletions
diff --git a/python/dpd/Adapt.py b/python/dpd/Adapt.py index 840aee9..a30f0c8 100644 --- a/python/dpd/Adapt.py +++ b/python/dpd/Adapt.py @@ -9,263 +9,184 @@ This module is used to change settings of ODR-DabMod using  the ZMQ remote control socket.  """ -import zmq  import logging  import numpy as np -import os -import datetime +import os.path  import pickle +from lib import zmqrc +from typing import List  LUT_LEN = 32  FORMAT_POLY = 1  FORMAT_LUT = 2 -def _write_poly_coef_file(coefs_am, coefs_pm, path): -    assert (len(coefs_am) == len(coefs_pm)) +def _write_poly_coef_file(coefs_am: List[float], coefs_pm: List[float], path: str) -> None: +    assert len(coefs_am) == len(coefs_pm) -    f = open(path, 'w') -    f.write("{}\n{}\n".format(FORMAT_POLY, len(coefs_am))) -    for coef in coefs_am: -        f.write("{}\n".format(coef)) -    for coef in coefs_pm: -        f.write("{}\n".format(coef)) -    f.close() +    with open(path, 'w') as f: +        f.write("{}\n{}\n".format(FORMAT_POLY, len(coefs_am))) +        for coef in coefs_am: +            f.write("{}\n".format(coef)) +        for coef in coefs_pm: +            f.write("{}\n".format(coef)) -def _write_lut_file(scalefactor, lut, path): -    assert (len(lut) == LUT_LEN) +def _write_lut_file(scalefactor: float, lut: List[complex], path: str) -> None: +    assert len(lut) == LUT_LEN -    f = open(path, 'w') -    f.write("{}\n{}\n".format(FORMAT_LUT, scalefactor)) -    for coef in lut: -        f.write("{}\n{}\n".format(coef.real, coef.imag)) -    f.close() +    with open(path, 'w') as f: +        f.write("{}\n{}\n".format(FORMAT_LUT, scalefactor)) +        for coef in lut: +            f.write("{}\n{}\n".format(coef.real, coef.imag)) -def dpddata_to_str(dpddata): +def dpddata_to_str(dpddata) -> str:      if dpddata[0] == "poly":          coefs_am = dpddata[1]          coefs_pm = dpddata[2] -        return "dpd_coefs_am {}, dpd_coefs_pm {}".format( +        return "Poly: AM/AM {}, AM/PM {}".format(                  coefs_am, coefs_pm)      elif dpddata[0] == "lut":          scalefactor = dpddata[1]          lut = dpddata[2] -        return "LUT scalefactor {}, LUT {}".format( +        return "LUT: scalefactor {}, LUT {}".format(                  scalefactor, lut)      else:          raise ValueError("Unknown dpddata type {}".format(dpddata[0]))  class Adapt: -    """Uses the ZMQ remote control to change parameters of the DabMod +    """Uses the ZMQ remote control to change parameters of the DabMod """ -    Parameters -    ---------- -    port : int -        Port at which the ODR-DabMod is listening to connect the -        ZMQ remote control. -    """ - -    def __init__(self, port, coef_path, plot_location): +    def __init__(self, port: int, coef_path: str, plot_location: str):          logging.debug("Instantiate Adapt object") -        self.port = port -        self.coef_path = coef_path -        self.plot_location = plot_location -        self.host = "localhost" -        self._context = zmq.Context() - -    def _connect(self): -        """Establish the connection to ODR-DabMod using -        a ZMQ socket that is in request mode (Client). -        Returns a socket""" -        sock = self._context.socket(zmq.REQ) -        poller = zmq.Poller() -        poller.register(sock, zmq.POLLIN) - -        sock.connect("tcp://%s:%d" % (self.host, self.port)) - -        sock.send(b"ping") - -        socks = dict(poller.poll(1000)) -        if socks: -            if socks.get(sock) == zmq.POLLIN: -                data = [el.decode() for el in sock.recv_multipart()] - -                if data != ['ok']: -                    raise RuntimeError( -                        "Invalid ZMQ RC answer to 'ping' at %s %d: %s" % -                        (self.host, self.port, data)) -        else: -            sock.close(linger=10) -            raise RuntimeError( -                    "ZMQ RC does not respond to 'ping' at %s %d" % -                        (self.host, self.port)) - -        return sock - -    def send_receive(self, message): -        """Send a message to ODR-DabMod. It always -        returns the answer ODR-DabMod sends back. +        self._port = port +        self._coef_path = coef_path +        self._plot_location = plot_location +        self._host = "localhost" +        self._mod_rc = zmqrc.ModRemoteControl(self._host, self._port) -        An example message could be -        "get sdr txgain" or "set sdr txgain 50" - -        Parameter -        --------- -        message : str -            The message string that will be sent to the receiver. -        """ -        sock = self._connect() -        logging.debug("Send message: %s" % message) -        msg_parts = message.split(" ") -        for i, part in enumerate(msg_parts): -            if i == len(msg_parts) - 1: -                f = 0 -            else: -                f = zmq.SNDMORE - -            sock.send(part.encode(), flags=f) - -        data = [el.decode() for el in sock.recv_multipart()] -        logging.debug("Received message: %s" % message) -        return data - -    def set_txgain(self, gain): -        """Set a new txgain for the ODR-DabMod. - -        Parameters -        ---------- -        gain : int -            new TX gain, in the same format as ODR-DabMod's config file -        """ +    def set_txgain(self, gain : float) -> None:          # TODO this is specific to the B200          if gain < 0 or gain > 89:              raise ValueError("Gain has to be in [0,89]") -        return self.send_receive("set sdr txgain %.4f" % float(gain)) +        self._mod_rc.set_param_value("sdr", "txgain", "%.4f" % float(gain)) -    def get_txgain(self): -        """Get the txgain value in dB for the ODR-DabMod.""" -        # TODO handle failure -        return float(self.send_receive("get sdr txgain")[0]) +    def get_txgain(self) -> float: +        """Get the txgain value in dB, or -1 in case of error""" +        try: +            return float(self._mod_rc.get_param_value("sdr", "txgain")) +        except ValueError as e: +            logging.warning(f"Adapt: get_txgain error: {e}") +            return -1.0 -    def set_rxgain(self, gain): -        """Set a new rxgain for the ODR-DabMod. - -        Parameters -        ---------- -        gain : int -            new RX gain, in the same format as ODR-DabMod's config file -        """ +    def set_rxgain(self, gain: float) -> None:          # TODO this is specific to the B200          if gain < 0 or gain > 89:              raise ValueError("Gain has to be in [0,89]") -        return self.send_receive("set sdr rxgain %.4f" % float(gain)) - -    def get_rxgain(self): -        """Get the rxgain value in dB for the ODR-DabMod.""" -        # TODO handle failure -        return float(self.send_receive("get sdr rxgain")[0]) - -    def set_digital_gain(self, gain): -        """Set a new rxgain for the ODR-DabMod. - -        Parameters -        ---------- -        gain : int -            new RX gain, in the same format as ODR-DabMod's config file -        """ -        msg = "set gain digital %.5f" % gain -        return self.send_receive(msg) - -    def get_digital_gain(self): -        """Get the rxgain value in dB for the ODR-DabMod.""" -        # TODO handle failure -        return float(self.send_receive("get gain digital")[0]) +        self._mod_rc.set_param_value("sdr", "rxgain", "%.4f" % float(gain)) + +    def get_rxgain(self) -> float: +        """Get the rxgain value in dB, or -1 in case of error""" +        try: +            return float(self._mod_rc.get_param_value("sdr", "rxgain")) +        except ValueError as e: +            logging.warning(f"Adapt: get_rxgain error: {e}") +            return -1.0 + +    def set_digital_gain(self, gain: float) -> None: +        self._mod_rc.set_param_value("gain", "digital", "%.5f" % float(gain)) + +    def get_digital_gain(self) -> float: +        """Get the digital gain value in linear scale, or -1 in case +        of error""" +        try: +            return float(self._mod_rc.get_param_value("gain", "digital")) +        except ValueError as e: +            logging.warning(f"Adapt: get_digital_gain error: {e}") +            return -1.0      def get_predistorter(self):          """Load the coefficients from the file in the format given in the README,          return ("poly", [AM coef], [PM coef]) or ("lut", scalefactor, [LUT entries])          """ -        f = open(self.coef_path, 'r') -        lines = f.readlines() -        predistorter_format = int(lines[0]) -        if predistorter_format == FORMAT_POLY: -            coefs_am_out = [] -            coefs_pm_out = [] -            n_coefs = int(lines[1]) -            coefs = [float(l) for l in lines[2:]] -            i = 0 -            for c in coefs: -                if i < n_coefs: -                    coefs_am_out.append(c) -                elif i < 2 * n_coefs: -                    coefs_pm_out.append(c) -                else: -                    raise ValueError( -                        'Incorrect coef file format: too many' -                        ' coefficients in {}, should be {}, coefs are {}' -                            .format(self.coef_path, n_coefs, coefs)) -                i += 1 -            f.close() -            return 'poly', coefs_am_out, coefs_pm_out -        elif predistorter_format == FORMAT_LUT: -            scalefactor = int(lines[1]) -            coefs = np.array([float(l) for l in lines[2:]], dtype=np.float32) -            coefs = coefs.reshape((-1, 2)) -            lut = coefs[..., 0] + 1j * coefs[..., 1] -            if len(lut) != LUT_LEN: -                raise ValueError("Incorrect number of LUT entries ({} expected {})".format(len(lut), LUT_LEN)) -            return 'lut', scalefactor, lut -        else: -            raise ValueError("Unknown predistorter format {}".format(predistorter_format)) +        with open(self._coef_path, 'r') as f: +            lines = f.readlines() +            predistorter_format = int(lines[0]) +            if predistorter_format == FORMAT_POLY: +                coefs_am_out = [] +                coefs_pm_out = [] +                n_coefs = int(lines[1]) +                coefs = [float(l) for l in lines[2:]] +                for i, c in enumerate(coefs): +                    if i < n_coefs: +                        coefs_am_out.append(c) +                    elif i < 2 * n_coefs: +                        coefs_pm_out.append(c) +                    else: +                        raise ValueError( +                            'Incorrect coef file format: too many' +                            ' coefficients in {}, should be {}, coefs are {}' +                                .format(self._coef_path, n_coefs, coefs)) +                return 'poly', coefs_am_out, coefs_pm_out +            elif predistorter_format == FORMAT_LUT: +                scalefactor = int(lines[1]) +                coefs = np.array([float(l) for l in lines[2:]], dtype=np.float32) +                coefs = coefs.reshape((-1, 2)) +                lut = coefs[..., 0] + 1j * coefs[..., 1] +                if len(lut) != LUT_LEN: +                    raise ValueError("Incorrect number of LUT entries ({} expected {})".format(len(lut), LUT_LEN)) +                return 'lut', scalefactor, lut +            else: +                raise ValueError("Unknown predistorter format {}".format(predistorter_format)) -    def set_predistorter(self, dpddata): +    def set_predistorter(self, dpddata) -> None:          """Update the predistorter data in the modulator. Takes the same          tuple format as argument than the one returned get_predistorter()"""          if dpddata[0] == "poly":              coefs_am = dpddata[1]              coefs_pm = dpddata[2] -            _write_poly_coef_file(coefs_am, coefs_pm, self.coef_path) +            _write_poly_coef_file(coefs_am, coefs_pm, self._coef_path)          elif dpddata[0] == "lut":              scalefactor = dpddata[1]              lut = dpddata[2] -            _write_lut_file(scalefactor, lut, self.coef_path) +            _write_lut_file(scalefactor, lut, self._coef_path)          else:              raise ValueError("Unknown predistorter '{}'".format(dpddata[0])) -        self.send_receive("set memlesspoly coeffile {}".format(self.coef_path)) +        self._mod_rc.set_param_value("memlesspoly", "coeffile", self._coef_path) -    def dump(self, path=None): +    def dump(self, path: str) -> None:          """Backup current settings to a file""" -        dt = datetime.datetime.now().isoformat() -        if path is None: -            if self.plot_location is not None: -                path = self.plot_location + "/" + dt + "_adapt.pkl" -            else: -                raise Exception("Cannot dump Adapt without either plot_location or path set") +          d = {              "txgain": self.get_txgain(),              "rxgain": self.get_rxgain(),              "digital_gain": self.get_digital_gain(), -            "predistorter": self.get_predistorter() +            "dpddata": self.get_predistorter()          } +          with open(path, "wb") as f:              pickle.dump(d, f) -        return path - -    def load(self, path): +    def restore(self, path: str):          """Restore settings from a file"""          with open(path, "rb") as f:              d = pickle.load(f) -            self.set_txgain(d["txgain"]) +            self.set_txgain(0) + +            # If any of the following fail, we will be running +            # with the safe value of txgain=0              self.set_digital_gain(d["digital_gain"])              self.set_rxgain(d["rxgain"]) -            self.set_predistorter(d["predistorter"]) +            self.set_predistorter(d["dpddata"]) +            self.set_txgain(d["txgain"]) + +            return d  # The MIT License (MIT)  # -# Copyright (c) 2017 Andreas Steger, Matthias P. Braendli +# Copyright (c) 2019 Matthias P. Braendli +# Copyright (c) 2017 Andreas Steger  #  # Permission is hereby granted, free of charge, to any person obtaining a copy  # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/ExtractStatistic.py b/python/dpd/ExtractStatistic.py index 639513a..3518dba 100644 --- a/python/dpd/ExtractStatistic.py +++ b/python/dpd/ExtractStatistic.py @@ -38,14 +38,16 @@ class ExtractStatistic:      """Calculate a low variance RX value for equally spaced tx values      of a predefined range""" -    def __init__(self, c): +    def __init__(self, c, peak_amplitude):          self.c = c +        self._plot_data = None +          # Number of measurements used to extract the statistic          self.n_meas = 0          # Boundaries for the bins -        self.tx_boundaries = np.linspace(c.ES_start, c.ES_end, c.ES_n_bins + 1) +        self.tx_boundaries = np.linspace(0.0, peak_amplitude, c.ES_n_bins + 1)          self.n_per_bin = c.ES_n_per_bin          # List of rx values for each bin @@ -58,12 +60,14 @@ class ExtractStatistic:          for i in range(c.ES_n_bins):              self.tx_values_lists.append([]) -        self.plot = c.ES_plot +    def get_bin_info(self): +        return "Binning: {} bins used for amplitudes between {} and {}".format( +                len(self.tx_boundaries), np.min(self.tx_boundaries), np.max(self.tx_boundaries)) + +    def plot(self, plot_path, title): +        if self._plot_data is not None: +            tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists = self._plot_data -    def _plot_and_log(self, tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists): -        if self.plot and self.c.plot_location is not None: -            dt = datetime.datetime.now().isoformat() -            fig_path = self.c.plot_location + "/" + dt + "_ExtractStatistic.png"              sub_rows = 3              sub_cols = 1              fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) @@ -72,7 +76,7 @@ class ExtractStatistic:              i_sub += 1              ax = plt.subplot(sub_rows, sub_cols, i_sub)              ax.plot(tx_values, rx_values, -                    label="Estimated Values", +                    label="Averaged measurements",                      color="red")              for i, tx_value in enumerate(tx_values):                  rx_values_list = self.rx_values_lists[i] @@ -80,17 +84,17 @@ class ExtractStatistic:                             np.abs(rx_values_list),                             s=0.1,                             color="black") -            ax.set_title("Extracted Statistic") +            ax.set_title("Extracted Statistic {}".format(title))              ax.set_xlabel("TX Amplitude")              ax.set_ylabel("RX Amplitude") -            ax.set_ylim(0, 0.8) -            ax.set_xlim(0, 1.1) +            ax.set_ylim(0, np.max(self.tx_boundaries)) # we expect a rougly a 1:1 correspondence between x and y +            ax.set_xlim(0, np.max(self.tx_boundaries))              ax.legend(loc=4)              i_sub += 1              ax = plt.subplot(sub_rows, sub_cols, i_sub)              ax.plot(tx_values, np.rad2deg(phase_diffs_values), -                    label="Estimated Values", +                    label="Averaged measurements",                      color="red")              for i, tx_value in enumerate(tx_values):                  phase_diff = phase_diffs_values_lists[i] @@ -101,7 +105,7 @@ class ExtractStatistic:              ax.set_xlabel("TX Amplitude")              ax.set_ylabel("Phase Difference")              ax.set_ylim(-60, 60) -            ax.set_xlim(0, 1.1) +            ax.set_xlim(0, np.max(self.tx_boundaries))              ax.legend(loc=4)              num = [] @@ -111,12 +115,12 @@ class ExtractStatistic:              i_sub += 1              ax = plt.subplot(sub_rows, sub_cols, i_sub)              ax.plot(num) -            ax.set_xlabel("TX Amplitude") +            ax.set_xlabel("TX Amplitude bin")              ax.set_ylabel("Number of Samples")              ax.set_ylim(0, self.n_per_bin * 1.2)              fig.tight_layout() -            fig.savefig(fig_path) +            fig.savefig(plot_path)              plt.close(fig)      def _rx_value_per_bin(self): @@ -166,7 +170,7 @@ class ExtractStatistic:          phase_diffs_values_lists = self._phase_diff_list_per_bin()          phase_diffs_values = _phase_diff_value_per_bin(phase_diffs_values_lists) -        self._plot_and_log(tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists) +        self._plot_data = (tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists)          tx_values_crop = np.array(tx_values, dtype=np.float32)[:idx_end]          rx_values_crop = np.array(rx_values, dtype=np.float32)[:idx_end] @@ -176,6 +180,7 @@ class ExtractStatistic:  # The MIT License (MIT)  #  # Copyright (c) 2017 Andreas Steger +# Copyright (c) 2018 Matthias P. Braendli  #  # Permission is hereby granted, free of charge, to any person obtaining a copy  # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/GlobalConfig.py b/python/dpd/GlobalConfig.py index 873b6ac..632a63b 100644 --- a/python/dpd/GlobalConfig.py +++ b/python/dpd/GlobalConfig.py @@ -10,7 +10,7 @@  import numpy as np  class GlobalConfig: -    def __init__(self, samplerate, plot_location: str): +    def __init__(self, samplerate: int, plot_location: str):          self.sample_rate = samplerate          assert self.sample_rate == 8192000, "We only support constants for 8192000 sample rate: {}".format(self.sample_rate) @@ -26,6 +26,8 @@ class GlobalConfig:          self.T_U = oversample * 2048  # Inverse of carrier spacing          self.T_C = oversample * 504  # Duration of cyclic prefix +        self.median_to_peak = 12 # Estimated value for a DAB OFDM signal +          # Frequency Domain          # example: np.delete(fft[3328:4865], 768)          self.FFT_delta = 1536  # Number of carrier frequencies @@ -40,10 +42,8 @@ class GlobalConfig:          self.phase_offset_per_sample = 1. / self.sample_rate * 2 * np.pi * 1000          # Constants for ExtractStatistic -        self.ES_plot = plot -        self.ES_start = 0.0          self.ES_end = 1.0 -        self.ES_n_bins = 64  # Number of bins between ES_start and ES_end +        self.ES_n_bins = 64          self.ES_n_per_bin = 128  # Number of measurements pre bin          # Constants for Measure_Shoulder @@ -68,9 +68,6 @@ class GlobalConfig:          # Constants for MER          self.MER_plot = plot -        # Constants for Model -        self.MDL_plot = plot -          # Constants for Model_PM          # Set all phase offsets to zero for TX amplitude < MPM_tx_min          self.MPM_tx_min = 0.1 diff --git a/python/dpd/Measure.py b/python/dpd/Measure.py index 489c4c0..e5a72c7 100644 --- a/python/dpd/Measure.py +++ b/python/dpd/Measure.py @@ -15,7 +15,7 @@ import logging  class Measure:      """Collect Measurement from DabMod""" -    def __init__(self, config, samplerate, port, num_samples_to_request): +    def __init__(self, config, samplerate : int, port : int, num_samples_to_request : int):          logging.info("Instantiate Measure object")          self.c = config          self.samplerate = samplerate @@ -23,7 +23,7 @@ class Measure:          self.port = port          self.num_samples_to_request = num_samples_to_request -    def _recv_exact(self, sock, num_bytes): +    def _recv_exact(self, sock : socket.socket, num_bytes : int) -> bytes:          """Receive an exact number of bytes from a socket. This is          a wrapper around sock.recv() that can return less than the number          of requested bytes. @@ -41,7 +41,7 @@ class Measure:              bufs.append(b)          return b''.join(bufs) -    def receive_tcp(self): +    def receive_tcp(self, num_samples_to_request : int):          s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)          s.settimeout(4)          s.connect(('localhost', self.port)) @@ -49,8 +49,8 @@ class Measure:          logging.debug("Send version")          s.sendall(b"\x01") -        logging.debug("Send request for {} samples".format(self.num_samples_to_request)) -        s.sendall(struct.pack("=I", self.num_samples_to_request)) +        logging.debug("Send request for {} samples".format(num_samples_to_request)) +        s.sendall(struct.pack("=I", num_samples_to_request))          logging.debug("Wait for TX metadata")          num_samps, tx_second, tx_pps = struct.unpack("=III", self._recv_exact(s, 12)) @@ -90,14 +90,35 @@ class Measure:          return txframe, tx_ts, rxframe, rx_ts +    def get_samples_unaligned(self, short=False): +        """Connect to ODR-DabMod, retrieve TX and RX samples, load +        into numpy arrays, and return a tuple +        (txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median) +        """ + +        n_samps = int(self.num_samples_to_request / 4) if short else self.num_samples_to_request +        txframe, tx_ts, rxframe, rx_ts = self.receive_tcp(n_samps) + +        # Normalize received signal with sent signal +        rx_median = np.median(np.abs(rxframe)) +        tx_median = np.median(np.abs(txframe)) +        rxframe = rxframe / rx_median * tx_median + + +        logging.info( +            "Measurement done, tx %d %s, rx %d %s" % +            (len(txframe), txframe.dtype, len(rxframe), rxframe.dtype)) + +        return txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median -    def get_samples(self): +    def get_samples(self, short=False):          """Connect to ODR-DabMod, retrieve TX and RX samples, load          into numpy arrays, and return a tuple          (txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median)          """ -        txframe, tx_ts, rxframe, rx_ts = self.receive_tcp() +        n_samps = int(self.num_samples_to_request / 4) if short else self.num_samples_to_request +        txframe, tx_ts, rxframe, rx_ts = self.receive_tcp(n_samps)          # Normalize received signal with sent signal          rx_median = np.median(np.abs(rxframe)) @@ -116,6 +137,7 @@ class Measure:  # The MIT License (MIT)  # +# Copyright (c) 2018 Matthias P. Braendli  # Copyright (c) 2017 Andreas Steger  #  # Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/python/dpd/Model_AM.py b/python/dpd/Model_AM.py deleted file mode 100644 index 75b226f..0000000 --- a/python/dpd/Model_AM.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- -# -# DPD Computation Engine, model implementation for Amplitude and not Phase -# -# http://www.opendigitalradio.org -# Licence: The MIT License, see notice at the end of this file - -import datetime -import os -import logging -import numpy as np -import matplotlib.pyplot as plt - - -def is_npfloat32(array): -    assert isinstance(array, np.ndarray), type(array) -    assert array.dtype == np.float32, array.dtype -    assert array.flags.contiguous -    assert not any(np.isnan(array)) - - -def check_input_get_next_coefs(tx_dpd, rx_received): -    is_npfloat32(tx_dpd) -    is_npfloat32(rx_received) - - -def poly(sig): -    return np.array([sig ** i for i in range(1, 6)]).T - - -def fit_poly(tx_abs, rx_abs): -    return np.linalg.lstsq(poly(rx_abs), tx_abs, rcond=None)[0] - - -def calc_line(coefs, min_amp, max_amp): -    rx_range = np.linspace(min_amp, max_amp) -    tx_est = np.sum(poly(rx_range) * coefs, axis=1) -    return tx_est, rx_range - - -class Model_AM: -    """Calculates new coefficients using the measurement and the previous -    coefficients""" - -    def __init__(self, -                 c, -                 learning_rate_am=1, -                 plot=False): -        self.c = c - -        self.learning_rate_am = learning_rate_am -        self.plot = plot - -    def _plot(self, tx_dpd, rx_received, coefs_am, coefs_am_new): -        if self.plot and self.c.plot_location is not None: -            tx_range, rx_est = calc_line(coefs_am, 0, 0.6) -            tx_range_new, rx_est_new = calc_line(coefs_am_new, 0, 0.6) - -            dt = datetime.datetime.now().isoformat() -            fig_path = self.c.plot_location + "/" + dt + "_Model_AM.png" -            sub_rows = 1 -            sub_cols = 1 -            fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) -            i_sub = 0 - -            i_sub += 1 -            ax = plt.subplot(sub_rows, sub_cols, i_sub) -            ax.plot(tx_range, rx_est, -                    label="Estimated TX", -                    alpha=0.3, -                    color="gray") -            ax.plot(tx_range_new, rx_est_new, -                    label="New Estimated TX", -                    color="red") -            ax.scatter(tx_dpd, rx_received, -                       label="Binned Data", -                       color="blue", -                       s=1) -            ax.set_title("Model_AM") -            ax.set_xlabel("TX Amplitude") -            ax.set_ylabel("RX Amplitude") -            ax.set_xlim(-0.5, 1.5) -            ax.legend(loc=4) - -            fig.tight_layout() -            fig.savefig(fig_path) -            plt.close(fig) - -    def get_next_coefs(self, tx_dpd, rx_received, coefs_am): -        """Calculate the next AM/AM coefficients using the extracted -        statistic of TX and RX amplitude""" -        check_input_get_next_coefs(tx_dpd, rx_received) - -        coefs_am_new = fit_poly(tx_dpd, rx_received) -        coefs_am_new = coefs_am + \ -                       self.learning_rate_am * (coefs_am_new - coefs_am) - -        self._plot(tx_dpd, rx_received, coefs_am, coefs_am_new) - -        return coefs_am_new - -# The MIT License (MIT) -# -# Copyright (c) 2017 Andreas Steger -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/python/dpd/Model_PM.py b/python/dpd/Model_PM.py deleted file mode 100644 index 7b80bf3..0000000 --- a/python/dpd/Model_PM.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- coding: utf-8 -*- -# -# DPD Computation Engine, model implementation for Amplitude and not Phase -# -# http://www.opendigitalradio.org -# Licence: The MIT License, see notice at the end of this file - -import datetime -import os -import logging -import numpy as np -import matplotlib.pyplot as plt - - -def is_npfloat32(array): -    assert isinstance(array, np.ndarray), type(array) -    assert array.dtype == np.float32, array.dtype -    assert array.flags.contiguous -    assert not any(np.isnan(array)) - - -def check_input_get_next_coefs(tx_dpd, phase_diff): -    is_npfloat32(tx_dpd) -    is_npfloat32(phase_diff) - - -class Model_PM: -    """Calculates new coefficients using the measurement and the previous -    coefficients""" - -    def __init__(self, -                 c, -                 learning_rate_pm=1, -                 plot=False): -        self.c = c - -        self.learning_rate_pm = learning_rate_pm -        self.plot = plot - -    def _plot(self, tx_dpd, phase_diff, coefs_pm, coefs_pm_new): -        if self.plot and self.c.plot_location is not None: -            tx_range, phase_diff_est = self.calc_line(coefs_pm, 0, 0.6) -            tx_range_new, phase_diff_est_new = self.calc_line(coefs_pm_new, 0, 0.6) - -            dt = datetime.datetime.now().isoformat() -            fig_path = self.c.plot_location + "/" + dt + "_Model_PM.png" -            sub_rows = 1 -            sub_cols = 1 -            fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) -            i_sub = 0 - -            i_sub += 1 -            ax = plt.subplot(sub_rows, sub_cols, i_sub) -            ax.plot(tx_range, phase_diff_est, -                    label="Estimated Phase Diff", -                    alpha=0.3, -                    color="gray") -            ax.plot(tx_range_new, phase_diff_est_new, -                    label="New Estimated Phase Diff", -                    color="red") -            ax.scatter(tx_dpd, phase_diff, -                       label="Binned Data", -                       color="blue", -                       s=1) -            ax.set_title("Model_PM") -            ax.set_xlabel("TX Amplitude") -            ax.set_ylabel("Phase DIff") -            ax.legend(loc=4) - -            fig.tight_layout() -            fig.savefig(fig_path) -            plt.close(fig) - -    def _discard_small_values(self, tx_dpd, phase_diff): -        """ Assumes that the phase for small tx amplitudes is zero""" -        mask = tx_dpd < self.c.MPM_tx_min -        phase_diff[mask] = 0 -        return tx_dpd, phase_diff - -    def poly(self, sig): -        return np.array([sig ** i for i in range(0, 5)]).T - -    def fit_poly(self, tx_abs, phase_diff): -        return np.linalg.lstsq(self.poly(tx_abs), phase_diff, rcond=None)[0] - -    def calc_line(self, coefs, min_amp, max_amp): -        tx_range = np.linspace(min_amp, max_amp) -        phase_diff = np.sum(self.poly(tx_range) * coefs, axis=1) -        return tx_range, phase_diff - -    def get_next_coefs(self, tx_dpd, phase_diff, coefs_pm): -        """Calculate the next AM/PM coefficients using the extracted -        statistic of TX amplitude and phase difference""" -        tx_dpd, phase_diff = self._discard_small_values(tx_dpd, phase_diff) -        check_input_get_next_coefs(tx_dpd, phase_diff) - -        coefs_pm_new = self.fit_poly(tx_dpd, phase_diff) - -        coefs_pm_new = coefs_pm + self.learning_rate_pm * (coefs_pm_new - coefs_pm) -        self._plot(tx_dpd, phase_diff, coefs_pm, coefs_pm_new) - -        return coefs_pm_new - -# The MIT License (MIT) -# -# Copyright (c) 2017 Andreas Steger -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/python/dpd/Model_Poly.py b/python/dpd/Model_Poly.py index c8f6135..ef3fed3 100644 --- a/python/dpd/Model_Poly.py +++ b/python/dpd/Model_Poly.py @@ -8,15 +8,13 @@  import os  import logging  import numpy as np +import matplotlib.pyplot as plt -import dpd.Model_AM as Model_AM -import dpd.Model_PM as Model_PM - - -def assert_np_float32(x): -    assert isinstance(x, np.ndarray) -    assert x.dtype == np.float32 -    assert x.flags.contiguous +def assert_np_float32(array): +    assert isinstance(array, np.ndarray), type(array) +    assert array.dtype == np.float32, array.dtype +    assert array.flags.contiguous +    assert not any(np.isnan(array))  def _check_input_get_next_coefs(tx_abs, rx_abs, phase_diff): @@ -36,20 +34,73 @@ class Poly:      """Calculates new coefficients using the measurement and the previous      coefficients""" -    def __init__(self, -                 c, -                 learning_rate_am=1.0, -                 learning_rate_pm=1.0): +    def __init__(self, c, learning_rate_am=1.0, learning_rate_pm=1.0):          self.c = c -        self.plot = c.MDL_plot          self.learning_rate_am = learning_rate_am          self.learning_rate_pm = learning_rate_pm          self.reset_coefs() -        self.model_am = Model_AM.Model_AM(c, plot=self.plot) -        self.model_pm = Model_PM.Model_PM(c, plot=self.plot) +    def plot(self, plot_location, title): +        if self._am_plot_data is not None and self._pm_plot_data is not None: +            tx_dpd, rx_received, coefs_am, coefs_am_new = self._am_plot_data + +            tx_range, rx_est = self._am_calc_line(coefs_am, 0, 0.6) +            tx_range_new, rx_est_new = self._am_calc_line(coefs_am_new, 0, 0.6) + +            sub_rows = 2 +            sub_cols = 1 +            fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) +            i_sub = 0 + +            # AM subplot +            i_sub += 1 +            ax = plt.subplot(sub_rows, sub_cols, i_sub) +            ax.plot(tx_range, rx_est, +                    label="Estimated TX", +                    alpha=0.3, +                    color="gray") +            ax.plot(tx_range_new, rx_est_new, +                    label="New Estimated TX", +                    color="red") +            ax.scatter(tx_dpd, rx_received, +                       label="Binned Data", +                       color="blue", +                       s=1) +            ax.set_title("Model AM and PM {}".format(title)) +            ax.set_xlabel("TX Amplitude") +            ax.set_ylabel("RX Amplitude") +            ax.set_xlim(0, 1.0) +            ax.legend(loc=4) + +            # PM sub plot +            tx_dpd, phase_diff, coefs_pm, coefs_pm_new = self._pm_plot_data + +            tx_range, phase_diff_est = self._pm_calc_line(coefs_pm, 0, 0.6) +            tx_range_new, phase_diff_est_new = self._pm_calc_line(coefs_pm_new, 0, 0.6) + +            i_sub += 1 +            ax = plt.subplot(sub_rows, sub_cols, i_sub) +            ax.plot(tx_range, phase_diff_est, +                    label="Estimated Phase Diff", +                    alpha=0.3, +                    color="gray") +            ax.plot(tx_range_new, phase_diff_est_new, +                    label="New Estimated Phase Diff", +                    color="red") +            ax.scatter(tx_dpd, phase_diff, +                       label="Binned Data", +                       color="blue", +                       s=1) +            ax.set_xlabel("TX Amplitude") +            ax.set_ylabel("Phase DIff") +            ax.set_xlim(0, 1.0) +            ax.legend(loc=4) + +            fig.tight_layout() +            fig.savefig(plot_location) +            plt.close(fig)      def reset_coefs(self):          self.coefs_am = np.zeros(5, dtype=np.float32) @@ -65,12 +116,8 @@ class Poly:          """          _check_input_get_next_coefs(tx_abs, rx_abs, phase_diff) -        if not lr is None: -            self.model_am.learning_rate_am = lr -            self.model_pm.learning_rate_pm = lr - -        coefs_am_new = self.model_am.get_next_coefs(tx_abs, rx_abs, self.coefs_am) -        coefs_pm_new = self.model_pm.get_next_coefs(tx_abs, phase_diff, self.coefs_pm) +        coefs_am_new = self._am_get_next_coefs(tx_abs, rx_abs, self.coefs_am) +        coefs_pm_new = self._pm_get_next_coefs(tx_abs, phase_diff, self.coefs_pm)          self.coefs_am = self.coefs_am + (coefs_am_new - self.coefs_am) * self.learning_rate_am          self.coefs_pm = self.coefs_pm + (coefs_pm_new - self.coefs_pm) * self.learning_rate_pm @@ -78,9 +125,67 @@ class Poly:      def get_dpd_data(self):          return "poly", self.coefs_am, self.coefs_pm +    def set_dpd_data(self, dpddata): +        if dpddata[0] != "poly" or len(dpddata) != 3: +            raise ValueError("dpddata is not of 'poly' format") +        _, self.coefs_am, self.coefs_pm = dpddata + +    def _am_calc_line(self, coefs, min_amp, max_amp): +        rx_range = np.linspace(min_amp, max_amp) +        tx_est = np.sum(self._am_poly(rx_range) * coefs, axis=1) +        return tx_est, rx_range + +    def _am_poly(self, sig): +        return np.array([sig ** i for i in range(1, 6)]).T + +    def _am_fit_poly(self, tx_abs, rx_abs): +        return np.linalg.lstsq(self._am_poly(rx_abs), tx_abs, rcond=None)[0] + +    def _am_get_next_coefs(self, tx_dpd, rx_received, coefs_am): +        """Calculate the next AM/AM coefficients using the extracted +        statistic of TX and RX amplitude""" + +        coefs_am_new = self._am_fit_poly(tx_dpd, rx_received) +        coefs_am_new = coefs_am + \ +                       self.learning_rate_am * (coefs_am_new - coefs_am) + +        self._am_plot_data = (tx_dpd, rx_received, coefs_am, coefs_am_new) + +        return coefs_am_new + +    def _pm_poly(self, sig): +        return np.array([sig ** i for i in range(0, 5)]).T + +    def _pm_calc_line(self, coefs, min_amp, max_amp): +        tx_range = np.linspace(min_amp, max_amp) +        phase_diff = np.sum(self._pm_poly(tx_range) * coefs, axis=1) +        return tx_range, phase_diff + +    def _discard_small_values(self, tx_dpd, phase_diff): +        """ Assumes that the phase for small tx amplitudes is zero""" +        mask = tx_dpd < self.c.MPM_tx_min +        phase_diff[mask] = 0 +        return tx_dpd, phase_diff + +    def _pm_fit_poly(self, tx_abs, phase_diff): +        return np.linalg.lstsq(self._pm_poly(tx_abs), phase_diff, rcond=None)[0] + +    def _pm_get_next_coefs(self, tx_dpd, phase_diff, coefs_pm): +        """Calculate the next AM/PM coefficients using the extracted +        statistic of TX amplitude and phase difference""" +        tx_dpd, phase_diff = self._discard_small_values(tx_dpd, phase_diff) + +        coefs_pm_new = self._pm_fit_poly(tx_dpd, phase_diff) + +        coefs_pm_new = coefs_pm + self.learning_rate_pm * (coefs_pm_new - coefs_pm) +        self._pm_plot_data = (tx_dpd, phase_diff, coefs_pm, coefs_pm_new) + +        return coefs_pm_new +  # The MIT License (MIT)  #  # Copyright (c) 2017 Andreas Steger +# Copyright (c) 2018 Matthias P. Brandli  #  # Permission is hereby granted, free of charge, to any person obtaining a copy  # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/RX_Agc.py b/python/dpd/RX_Agc.py index 48ef7f3..bb940be 100644 --- a/python/dpd/RX_Agc.py +++ b/python/dpd/RX_Agc.py @@ -19,19 +19,19 @@ import dpd.Adapt as Adapt  import dpd.Measure as Measure  class Agc: -    """The goal of the automatic gain control is to set the  -    RX gain to a value at which all received amplitudes can  -    be detected. This means that the maximum possible amplitude  +    """The goal of the automatic gain control is to set the +    RX gain to a value at which all received amplitudes can +    be detected. This means that the maximum possible amplitude      should be quantized at the highest possible digital value. -    A problem we have to face, is that the estimation of the  -    maximum amplitude by applying the max() function is very  -    unstable. This is due to the maximum’s rareness. Therefore  -    we estimate a far more robust value, such as the median,  +    A problem we have to face, is that the estimation of the +    maximum amplitude by applying the max() function is very +    unstable. This is due to the maximum’s rareness. Therefore +    we estimate a far more robust value, such as the median,      and then approximate the maximum amplitude from it. -    Given this, we tune the RX gain in such a way, that the  -    received signal fulfills our desired property, of having  +    Given this, we tune the RX gain in such a way, that the +    received signal fulfills our desired property, of having      all amplitudes properly quantized."""      def __init__(self, measure, adapt, c): @@ -45,10 +45,15 @@ class Agc:          self.peak_to_median = 1./c.RAGC_rx_median_target      def run(self) -> Tuple[bool, str]: -        self.adapt.set_rxgain(self.rxgain) +        try: +            self.adapt.set_rxgain(self.rxgain) +        except ValueError as e: +            return (False, "Setting RX gain to {} failed: {}".format(self.rxgain, e)) +        time.sleep(0.5) +          # Measure -        txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = self.measure.get_samples() +        txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median = self.measure.get_samples_unaligned(short=False)          # Estimate Maximum          rx_peak = self.peak_to_median * rx_median @@ -68,11 +73,19 @@ class Agc:              w = "Warning: calculated RX Gain={} is higher than maximum={}. RX feedback power should be increased.".format(                  self.rxgain, self.max_rxgain)              logging.warning(w) +            try: +                # Reset to a low value, as we expect the user to reduce external attenuation +                self.adapt.set_rxgain(30) +            except ValueError as e: +                return (False, "\n".join([measurements, w, "Setting RX gain to {} failed: {}".format(self.rxgain, e)]))              return (False, "\n".join([measurements, w]))          else: -            self.adapt.set_rxgain(self.rxgain) +            try: +                self.adapt.set_rxgain(self.rxgain) +            except ValueError as e: +                return (False, "Setting RX gain to {} failed: {}".format(self.rxgain, e))              time.sleep(0.5) -        return (True, measurements) +            return (True, measurements)      def plot_estimates(self):          """Plots the estimate of for Max, Median, Mean for different diff --git a/python/dpdce.py b/python/dpdce.py index 838d265..cf98aa0 100755 --- a/python/dpdce.py +++ b/python/dpdce.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3  # -*- coding: utf-8 -*-  #  # DPD Computation Engine standalone main file. @@ -43,7 +43,8 @@ rc_port = config.getint('rc_port')  samplerate = config.getint('samplerate')  samps = config.getint('samps')  coef_file = config['coef_file'] -log_folder = config['log_folder'] +logs_directory = config['logs_directory'] +plot_directory = config['plot_directory']  import logging  import datetime @@ -52,7 +53,7 @@ save_logs = False  # Simple usage scenarios don't need to clutter /tmp  if save_logs: -    dt = datetime.datetime.now().isoformat() +    dt = datetime.datetime.utcnow().isoformat()      logging_path = '/tmp/dpd_{}'.format(dt).replace('.', '_').replace(':', '-')      print("Logs and plots written to {}".format(logging_path))      os.makedirs(logging_path) @@ -71,7 +72,7 @@ if save_logs:      # add the handler to the root logger      logging.getLogger('').addHandler(console)  else: -    dt = datetime.datetime.now().isoformat() +    dt = datetime.datetime.utcnow().isoformat()      logging.basicConfig(format='%(asctime)s - %(module)s - %(levelname)s - %(message)s',                          datefmt='%Y-%m-%d %H:%M:%S',                          level=logging.INFO) @@ -79,10 +80,14 @@ else:  logging.info("DPDCE starting up"); +import time  import socket  from lib import yamlrpc  import numpy as np  import traceback +import os.path +import glob +import re  from threading import Thread, Lock  from queue import Queue  from dpd.Model import Poly @@ -96,13 +101,15 @@ from dpd.GlobalConfig import GlobalConfig  from dpd.MER import MER  from dpd.Measure_Shoulders import Measure_Shoulders -c = GlobalConfig(samplerate, logging_path) +plot_path = os.path.realpath(plot_directory) +coef_file = os.path.realpath(config['coef_file']) + +c = GlobalConfig(samplerate, plot_path)  symbol_align = Symbol_align(c)  mer = MER(c)  meas_shoulders = Measure_Shoulders(c)  meas = Measure(c, samplerate, dpd_port, samps) -extStat = ExtractStatistic(c) -adapt = Adapt(rc_port, coef_file, logging_path) +adapt = Adapt(rc_port, coef_file, plot_path)  model = Poly(c) @@ -128,54 +135,271 @@ if cli_args.reset:  cmd_socket = yamlrpc.Socket(bind_port=control_port)  # The following is accessed by both threads and need to be locked -settings = { -        'rx_gain': rx_gain, -        'tx_gain': tx_gain, -        'digital_gain': digital_gain, -        'dpddata': dpddata, +internal_data = { +        'n_runs': 0,          }  results = { +        'adapt_dumps': [], +        'statplot': None, +        'modelplot': None, +        'modeldata': dpddata_to_str(dpddata),          'tx_median': 0,          'rx_median': 0,          'state': 'Idle', +        'stateprogress': 0, # in percent          'summary': ['DPD has not been calibrated yet'],          }  lock = Lock()  command_queue = Queue(maxsize=1) +# Fill list of adapt dumps so that user can choose a previous +# setting across restarts. +results['adapt_dumps'].append("defaults") + +adapt_dump_files = glob.glob(os.path.join(plot_path, "adapt_*.pkl")) +re_adaptfile = re.compile(r"adapt_(.*)\.pkl") +for f in adapt_dump_files: +    match = re_adaptfile.search(f) +    if match: +        results['adapt_dumps'].append(match.group(1)) +  # Automatic Gain Control for the RX gain  agc = Agc(meas, adapt, c) +def clear_pngs(results): +    results['statplot'] = None +    results['modelplot'] = None +    pngs = glob.glob(os.path.join(plot_path, "*.png")) +    for png in pngs: +        try: +            os.remove(png) +        except: +            results['summary'] += ["failed to delete " + png] +  def engine_worker(): -    try: -        while True: +    extStat = None +    while True: +        try:              cmd = command_queue.get()              if cmd == "quit":                  break              elif cmd == "calibrate":                  with lock: -                    results['state'] = 'rx gain calibration' +                    results['state'] = 'RX Gain Calibration' +                    results['stateprogress'] = 0 +                    clear_pngs(results) -                agc_success, agc_summary = agc.run() -                summary = ["First calibration run:"] + agc_summary.split("\n") -                if agc_success: +                summary = [] +                N_ITER = 3 +                for i in range(N_ITER):                      agc_success, agc_summary = agc.run() -                    summary += ["Second calibration run: "] + agc_summary.split("\n") +                    summary += ["Iteration {}:".format(i)] + agc_summary.split("\n") + +                    with lock: +                        results['stateprogress'] = int((i + 1) * 100/N_ITER) +                        results['summary'] = ["Calibration ongoing:"] + summary + +                    if not agc_success: +                        break                  txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples()                  with lock: -                    settings['rx_gain'] = adapt.get_rxgain() -                    settings['digital_gain'] = adapt.get_digital_gain()                      results['tx_median'] = float(tx_median)                      results['rx_median'] = float(rx_median)                      results['state'] = 'Idle' -                    results['summary'] = ["Calibration was done:"] + summary +                    results['stateprogress'] = 100 +                    results['summary'] = summary + ["Calibration done"] +            elif cmd == "reset": +                model.reset_coefs() +                with lock: +                    internal_data['n_runs'] = 0 +                    results['state'] = 'Idle' +                    results['stateprogress'] = 0 +                    results['summary'] = ["Reset"] +                    results['modeldata'] = dpddata_to_str(model.get_dpd_data()) +                    clear_pngs(results) +                extStat = None +            elif cmd == "trigger_run": +                with lock: +                    results['state'] = 'Capture + Model' +                    results['stateprogress'] = 0 +                    n_runs = internal_data['n_runs'] + +                while True: +                    # Get Samples and check gain +                    txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples() + +                    if extStat is None: +                        # At first run, we must decide how to create the bins +                        peak_estimated = tx_median * c.median_to_peak +                        extStat = ExtractStatistic(c, peak_estimated) + +                    with lock: +                        results['stateprogress'] += 2 + +                    # Extract usable data from measurement +                    tx, rx, phase_diff, n_per_bin = extStat.extract(txframe_aligned, rxframe_aligned) + +                    utctime = datetime.datetime.utcnow() +                    plot_file = "stats_{}.png".format(utctime.strftime("%s")) +                    extStat.plot(os.path.join(plot_path, plot_file), utctime.strftime("%Y-%m-%dT%H%M%S")) +                    n_meas = Heuristics.get_n_meas(n_runs) + +                    with lock: +                        results['statplot'] = "dpd/" + plot_file +                        results['stateprogress'] += 2 +                        results['summary'] = ["Captured {} samples".format(len(txframe_aligned)), +                            "TX/RX median: {} / {}".format(tx_median, rx_median), +                            extStat.get_bin_info(), +                            "Extracted Statistics: TX median={} RX median={}".format(tx_median, rx_median), +                            "Runs: {}/{}".format(extStat.n_meas, n_meas)] +                    if extStat.n_meas >= n_meas: +                        break + +                if any(x is None for x in [tx, rx, phase_diff]): +                    with lock: +                        results['summary'] += ["Error! No data to calculate model"] +                        results['state'] = 'Idle' +                        results['stateprogress'] = 0 +                else: +                    with lock: +                        results['state'] = 'Capture + Model' +                        results['stateprogress'] = 80 +                        results['summary'] += ["Training model"] + +                    model.train(tx, rx, phase_diff, lr=Heuristics.get_learning_rate(n_runs)) + +                    utctime = datetime.datetime.utcnow() +                    model_plot_file = "model_{}.png".format(utctime.strftime("%s")) +                    model.plot( +                            os.path.join(plot_path, model_plot_file), +                            utctime.strftime("%Y-%m-%dT%H%M%S")) + +                    with lock: +                        results['modelplot'] = "dpd/" + model_plot_file +                        results['state'] = 'Capture + Model' +                        results['stateprogress'] = 85 +                        results['summary'] += ["Getting DPD data"] + +                    dpddata = model.get_dpd_data() +                    with lock: +                        internal_data['dpddata'] = dpddata +                        internal_data['n_runs'] = 0 + +                        results['modeldata'] = dpddata_to_str(dpddata) +                        results['state'] = 'Capture + Model' +                        results['stateprogress'] = 90 +                        results['summary'] += ["Reset statistics"] + +                    extStat = None + +                    with lock: +                        results['state'] = 'Idle' +                        results['stateprogress'] = 100 +                        results['summary'] += ["New DPD coefficients calculated"] +            elif cmd == "adapt": +                with lock: +                    dpddata = internal_data['dpddata'] +                    results['state'] = 'Update Predistorter' +                    results['stateprogress'] = 50 +                    results['summary'] = [""] +                    iteration = internal_data['n_runs'] +                    internal_data['n_runs'] += 1 + +                adapt.set_predistorter(dpddata) -    finally: -        with lock: -            results['state'] = 'terminated' +                time.sleep(2) + +                txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples() + +                # Store all settings for pre-distortion, tx and rx +                utctime = datetime.datetime.utcnow() +                dump_file = "adapt_{}.pkl".format(utctime.strftime("%s")) +                adapt.dump(os.path.join(plot_path, dump_file)) + +                with lock: +                    results['adapt_dumps'].append(utctime.strftime("%s")) + +                # Collect logging data +                off = symbol_align.calc_offset(txframe_aligned) +                tx_mer = mer.calc_mer(txframe_aligned[off:off + c.T_U], debug_name='TX') +                rx_mer = mer.calc_mer(rxframe_aligned[off:off + c.T_U], debug_name='RX') +                mse = np.mean(np.abs((txframe_aligned - rxframe_aligned) ** 2)) +                tx_gain = adapt.get_txgain() +                rx_gain = adapt.get_rxgain() +                digital_gain = adapt.get_digital_gain() +                rx_shoulder_tuple = meas_shoulders.average_shoulders(rxframe_aligned) +                tx_shoulder_tuple = meas_shoulders.average_shoulders(txframe_aligned) + +                lr = Heuristics.get_learning_rate(iteration) + +                summary = [f"Set predistorter:", +                        f"Signal measurements after iteration {iteration} with learning rate {lr}", +                        f"TX MER {tx_mer:.2}, RX MER {rx_mer:.2}", +                        f"Mean-square error: {mse:.3}"] +                if tx_shoulder_tuple is not None: +                    summary.append("Shoulders: TX {!r}, RX {!r}".format(tx_shoulder_tuple, rx_shoulder_tuple)) +                summary.append(f"Running with digital gain {digital_gain}, TX gain {tx_gain} and RX gain {rx_gain}") + +                with lock: +                    results['state'] = 'Update Predistorter' +                    results['stateprogress'] = 100 +                    results['summary'] = ["Signal measurements after predistortion update"] + summary +            elif cmd.startswith("restore_dump-"): +                _, _, dump_id = cmd.partition("-") +                if dump_id == "defaults": +                    model.reset_coefs() +                    dpddata = model.get_dpd_data() +                    adapt.set_predistorter(dpddata) + +                    tx_gain = adapt.get_txgain() +                    rx_gain = adapt.get_rxgain() +                    digital_gain = adapt.get_digital_gain() +                    with lock: +                        results['state'] = 'Idle' +                        results['stateprogress'] = 100 +                        results['summary'] = [f"Restored DPD defaults", +                            f"Running with digital gain {digital_gain}, TX gain {tx_gain} and RX gain {rx_gain}"] +                        results['modeldata'] = dpddata_to_str(dpddata) +                else: +                    dump_file = os.path.join(plot_path, f"adapt_{dump_id}.pkl") +                    try: +                        d = adapt.restore(dump_file) +                        logging.info(f"Restore: {d}") +                        model.set_dpd_data(d['dpddata']) +                        with lock: +                            results['state'] = 'Idle' +                            results['stateprogress'] = 100 +                            results['summary'] = [f"Restored DPD settings from dumpfile {dump_id}", +                                f"Running with digital gain {d['digital_gain']}, TX gain {d['txgain']} and RX gain {d['rxgain']}"] +                            results['modeldata'] = dpddata_to_str(d["dpddata"]) +                    except: +                        e = traceback.format_exc() +                        with lock: +                            results['state'] = 'Idle' +                            results['stateprogress'] = 100 +                            results['summary'] = [f"Failed to restore DPD settings from dumpfile {dump_id}", +                                    f"Error: {e}"] +        except: +            e = traceback.format_exc() +            logging.error(e) +            with lock: +                results['summary'] = [f"Exception:"] + e.split("\n") +                results['state'] = 'Autorestart pending' +                results['stateprogress'] = 0 + +            for i in range(5): +                time.sleep(2) +                with lock: +                    results['stateprogress'] += 20 +            time.sleep(2) +            with lock: +                dt = datetime.datetime.utcnow().isoformat() +                results['summary'] = [f"DPD engine auto-restarted at {dt} UTC", f"After exception {e}"] +                results['state'] = 'Idle' +                results['stateprogress'] = 0  engine = Thread(target=engine_worker) @@ -186,7 +410,7 @@ try:          try:              addr, msg_id, method, params = cmd_socket.receive_request()          except ValueError as e: -            logging.warning('YAML-RPC request error: {}'.format(e)) +            logging.warning('RPC request error: {}'.format(e))              continue          except TimeoutError:              continue @@ -194,28 +418,24 @@ try:              logging.info('Caught KeyboardInterrupt')              break          except: -            logging.error('YAML-RPC unknown error') +            logging.error('RPC unknown error')              break -        if method == 'trigger_run': -            logging.info('YAML-RPC request : {}'.format(method)) -            command_queue.put('trigger_run') -        elif method == 'reset': -            logging.info('YAML-RPC request : {}'.format(method)) -            command_queue.put('reset') -        elif method == 'set_setting': -            logging.info('YAML-RPC request : {} -> {}'.format(method, params)) -            # params == {'setting': ..., 'value': ...} -            pass -        elif method == 'get_settings': -            with lock: -                cmd_socket.send_success_response(addr, msg_id, settings) +        if any(method == m for m in ['trigger_run', 'reset', 'adapt']): +            logging.info('Received RPC request : {}'.format(method)) +            command_queue.put(method) +            cmd_socket.send_success_response(addr, msg_id, None) +        elif method == 'restore_dump': +            logging.info('Received RPC request : restore_dump({})'.format(params['dump_id'])) +            command_queue.put(f"restore_dump-{params['dump_id']}") +            cmd_socket.send_success_response(addr, msg_id, None)          elif method == 'get_results':              with lock:                  cmd_socket.send_success_response(addr, msg_id, results)          elif method == 'calibrate': -            logging.info('YAML-RPC request : {}'.format(method)) +            logging.info('Received RPC request : {}'.format(method))              command_queue.put('calibrate') +            cmd_socket.send_success_response(addr, msg_id, None)          else:              cmd_socket.send_error_response(addr, msg_id, "request not understood")  finally: @@ -346,7 +566,7 @@ while i < num_iter:  # The MIT License (MIT)  #  # Copyright (c) 2017 Andreas Steger -# Copyright (c) 2018 Matthias P. Braendli +# Copyright (c) 2019 Matthias P. Braendli  #  # Permission is hereby granted, free of charge, to any person obtaining a copy  # of this software and associated documentation files (the "Software"), to deal diff --git a/python/gui-dpdce.ini b/python/gui-dpdce.ini index 48c6abf..4385c80 100644 --- a/python/gui-dpdce.ini +++ b/python/gui-dpdce.ini @@ -19,7 +19,10 @@ samps=81920  coef_file=poly.coef  # Write logs to this folder, or leave empty for no logs -log_folder= +logs_directory= + +# Saving plots to the static directory makes them accessible to the browser +plot_directory=gui/static/dpd  [gui] diff --git a/python/gui.py b/python/gui.py index ce7948c..a9328ee 100755 --- a/python/gui.py +++ b/python/gui.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3  # -*- coding: utf-8 -*-  #  #   Copyright (C) 2018 @@ -33,6 +33,7 @@ from lib import zmqrc  env = Environment(loader=FileSystemLoader('gui/templates'))  base_js = ["js/odr.js"] +base_css = ["css/odr.css"]  class Root:      def __init__(self, dpd_port): @@ -51,7 +52,8 @@ class Root:      @cherrypy.expose      def home(self):          tmpl = env.get_template("home.html") -        return tmpl.render(tab='home', js=base_js, is_login=False) +        js = base_js + ["js/odr-home.js"] +        return tmpl.render(tab='home', js=js, css=base_css, is_login=False)      @cherrypy.expose      def rcvalues(self): diff --git a/python/gui/api.py b/python/gui/api.py index bff224e..f9e0ad0 100755 --- a/python/gui/api.py +++ b/python/gui/api.py @@ -1,6 +1,6 @@  # -*- coding: utf-8 -*-  # -#   Copyright (C) 2018 +#   Copyright (C) 2019  #   Matthias P. Braendli, matthias.braendli@mpb.li  #  #    http://www.opendigitalradio.org @@ -80,8 +80,18 @@ class API:                  return send_error(str(e))              return send_ok()          else: -            cherrypy.response.status = 400 -            return send_error("POST only") +            if all(p in kwargs for p in ('controllable', 'param')): +                try: +                    return send_ok(self.mod_rc.get_param_value(kwargs['controllable'], kwargs['param'])) +                except IOError as e: +                    cherrypy.response.status = 503 +                    return send_error(str(e)) +                except ValueError as e: +                    cherrypy.response.status = 503 +                    return send_error(str(e)) +            else: +                cherrypy.response.status = 400 +                return send_error("missing 'controllable' or 'param' GET parameters")      def _wrap_dpd(self, method, data=None):          try: @@ -89,12 +99,12 @@ class API:              return send_ok(reply)          except ValueError as e:              cherrypy.response.status = 503 -            return send_error("YAML-RPC call error: {}".format(e)) +            return send_error("DPDCE remote procedure call error: {}".format(e))          except TimeoutError as e:              cherrypy.response.status = 503 -            return send_error("YAML-RPC timeout: {}".format(e)) +            return send_error("DPDCE remote procedure call timed out")          cherrypy.response.status = 500 -        return send_error("YAML-RPC unknown error") +        return send_error("Unknown DPDCE remote procedure error error")      @cherrypy.expose      @cherrypy.tools.json_out() @@ -107,6 +117,15 @@ class API:      @cherrypy.expose      @cherrypy.tools.json_out() +    def dpd_adapt(self, **kwargs): +        if cherrypy.request.method == 'POST': +            return self._wrap_dpd("adapt") +        else: +            cherrypy.response.status = 400 +            return send_error("POST only") + +    @cherrypy.expose +    @cherrypy.tools.json_out()      def dpd_reset(self, **kwargs):          if cherrypy.request.method == 'POST':              return self._wrap_dpd("reset") @@ -116,12 +135,20 @@ class API:      @cherrypy.expose      @cherrypy.tools.json_out() -    def dpd_settings(self, setting: str, value: str, **kwargs): +    def dpd_restore_dump(self, **kwargs):          if cherrypy.request.method == 'POST': -            data = {'setting': setting, 'value': value} -            return self._wrap_dpd("set_setting", data) +            cl = cherrypy.request.headers['Content-Length'] +            rawbody = cherrypy.request.body.read(int(cl)) +            params = json.loads(rawbody.decode()) +            if 'dump_id' in params: +                data = {'dump_id': params['dump_id']} +                return self._wrap_dpd("restore_dump", data) +            else: +                cherrypy.response.status = 400 +                return send_error("Missing dump_id")          else: -            return self._wrap_dpd("get_settings") +            cherrypy.response.status = 400 +            return send_error("POST only")      @cherrypy.expose      @cherrypy.tools.json_out() diff --git a/python/gui/static/css/odr.css b/python/gui/static/css/odr.css new file mode 100644 index 0000000..1710464 --- /dev/null +++ b/python/gui/static/css/odr.css @@ -0,0 +1,14 @@ +.glyphicon-refresh-animate { +    -animation: spin 1.8s infinite linear; +    -webkit-animation: spin2 1.8s infinite linear; +} + +@-webkit-keyframes spin2 { +    from { -webkit-transform: rotate(0deg);} +    to { -webkit-transform: rotate(360deg);} +} + +@keyframes spin { +    from { transform: scale(1) rotate(0deg);} +    to { transform: scale(1) rotate(360deg);} +} diff --git a/python/gui/static/js/odr-home.js b/python/gui/static/js/odr-home.js new file mode 100644 index 0000000..b74c4c8 --- /dev/null +++ b/python/gui/static/js/odr-home.js @@ -0,0 +1,240 @@ +//   Copyright (C) 2019 +//   Matthias P. Braendli, matthias.braendli@mpb.li +// +//    http://www.opendigitalradio.org +// +//   This file is part of ODR-DabMod. +// +//   ODR-DabMod is free software: you can redistribute it and/or modify +//   it under the terms of the GNU General Public License as +//   published by the Free Software Foundation, either version 3 of the +//   License, or (at your option) any later version. +// +//   ODR-DabMod is distributed in the hope that it will be useful, +//   but WITHOUT ANY WARRANTY; without even the implied warranty of +//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +//   GNU General Public License for more details. +// +//   You should have received a copy of the GNU General Public License +//   along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +// + +function apiRequestChain(uri, get_data, success_callback, fail_callback) { +    $.ajax({ +        type: "GET", +        url: uri, +        data: get_data, +        contentType: 'application/json', +        dataType: 'json', + +        error: function(data) { +            console.log("GET " + JSON.stringify(get_data) + " error: " + data.responseText); +            fail_callback(data.responseText); +        }, +        success: function(data) { +            if (data.status == 'ok') { +                success_callback(data.data); +            } +            else { +                fail_callback(data.data); +            } +        }, +    }); +} + +function mark_pending(id, comment) { +    document.getElementById(id).className = "glyphicon glyphicon-refresh glyphicon-refresh-animate"; + +    if (comment) { +        document.getElementById(id + "_comment").innerHTML = comment; +    } +} + +var failure_encountered = false; + +function mark_overall_ok() { +    if (!failure_encountered) { +        document.getElementById("overall_state").className = "glyphicon glyphicon-ok"; +    } +} + +function mark_ok(id, comment) { +    document.getElementById(id).className = "glyphicon glyphicon-ok"; + +    if (comment) { +        document.getElementById(id + "_comment").innerHTML = comment; +    } +} + +function mark_fail(id, reason) { +    failure_encountered = true; + +    var el = document.getElementById(id); +    el.className = "glyphicon glyphicon-remove"; +    el.style.color = "#FF3333"; + +    document.getElementById(id + "_comment").innerHTML = reason; + +    var overall = document.getElementById("overall_state"); +    overall.style.color = "#FF8833"; +    overall.className = "glyphicon glyphicon-alert"; +} + +function check_rc() { +    mark_pending('is_rc_ok'); +    apiRequestChain("/api/parameter", +        {controllable: 'sdr', param: 'freq'}, +        function(data) { +            mark_ok('is_rc_ok'); +            check_modulating(0); +        }, +        function(data) { +            mark_fail('is_rc_ok', JSON.parse(data)['reason']); +        }); +} + +function check_modulating(last_num_frames) { +    mark_pending('is_modulating'); +    apiRequestChain("/api/parameter", +        {controllable: 'sdr', param: 'frames'}, +        function(data) { +            if (data > 0) { +                if (last_num_frames == 0) { +                    setTimeout(function() { check_modulating(data); }, 200); +                } +                else { +                    if (data == last_num_frames) { +                        mark_fail('is_modulating', "Frame counter not incrementing: " + data); +                    } +                    else { +                        mark_ok('is_modulating', "Number of frames modulated: " + data); +                    } +                    check_gpsdo_ok(); +                } +            } +            else { +                mark_fail('is_modulating', 'number of frames is 0'); +            } +        }, +        function(data) { +            mark_fail('is_modulating', data); +        }); +} + +function check_gpsdo_ok() { +    mark_pending('is_gpsdo_ok'); +    apiRequestChain("/api/parameter", +        {controllable: 'sdr', param: 'gpsdo_num_sv'}, +        function(data) { +            if (data > 3) { +                mark_ok('is_gpsdo_ok', "Number of SVs used: " + data); +            } +            else { +                mark_fail('is_gpsdo_ok', "Number of SVs (" + data + ") is too low"); +            } +            check_underrunning(0, 0); +            check_late(0, 0); +        }, +        function(data) { +            mark_fail('is_gpsdo_ok', json.parse(data)['reason']); +        }); +} + + +function check_underrunning(iteration, first_underruns) { +    var n_checks = 3; + +    apiRequestChain("/api/parameter", +        {controllable: 'sdr', param: 'underruns'}, +        function(data) { +            if (iteration == 0) { +                mark_pending('is_underrunning', "Checking for underruns"); +                setTimeout(function() { check_underrunning(iteration+1, data); }, 2000); +            } +            else if (iteration < n_checks) { +                mark_pending('is_underrunning', "Check " + iteration + "/" + n_checks + "..."); +                setTimeout(function() { check_underrunning(iteration+1, first_underruns); }, 2000); +            } +            else { +                if (data == first_underruns) { +                    mark_ok('is_underrunning', "Number of underruns is not increasing: " + data); +                } +                else { +                    mark_fail('is_underrunning', "Underruns observed in last " + n_checks + " seconds: " + data); +                } +                check_rate_4x(); +            } +        }, +        function(data) { +            mark_fail('is_underrunning', data); +        }); +} + +function check_late(iteration, first_late) { +    var n_checks = 3; + +    apiRequestChain("/api/parameter", +        {controllable: 'sdr', param: 'latepackets'}, +        function(data) { +            if (iteration == 0) { +                mark_pending('is_late', "Checking for late packets"); +                setTimeout(function() { check_late(iteration+1, data); }, 2000); +            } +            else if (iteration < n_checks) { +                mark_pending('is_late', "Check " + iteration + "/" + n_checks + "..."); +                setTimeout(function() { check_late(iteration+1, first_late); }, 2000); +            } +            else { +                if (data == first_late) { +                    mark_ok('is_late', "Number of late packets is not increasing: " + data); +                } +                else { +                    mark_fail('is_late', "Late packets observed in last " + n_checks + " seconds: " + data); +                } +            } +        }, +        function(data) { +            mark_fail('is_late', data); +        }); +} + +function check_rate_4x() { +    mark_pending('is_rate_4x'); +    apiRequestChain("/api/parameter", +        {controllable: 'modulator', param: 'rate'}, +        function(data) { +            if (data == 8192000) { +                mark_ok('is_rate_4x', "Samplerate: " + data); +            } +            else { +                mark_fail('is_rate_4x', "Samplerate is not 8192ksps: " + data); +            } +            check_dpdce_running(); +        }, +        function(data) { +            mark_fail('is_rate_4x', JSON.parse(data)['reason']); +        }); +} + +function check_dpdce_running() { +    mark_pending('is_dpdce_running'); +    apiRequestChain("/api/dpd_results", +        {}, +        function(data) { +            mark_ok('is_dpdce_running', "State: " + data['state']); +            mark_overall_ok(); +        }, +        function(data) { +            mark_fail('is_dpdce_running', JSON.parse(data)['reason']); +        }); +} + +$(function(){ +    setTimeout(check_rc, 20); +}); + + +// ToolTip init +$(function(){ +    $('[data-toggle="tooltip"]').tooltip(); +}); diff --git a/python/gui/static/js/odr-predistortion.js b/python/gui/static/js/odr-predistortion.js index 04d2773..4dae068 100644 --- a/python/gui/static/js/odr-predistortion.js +++ b/python/gui/static/js/odr-predistortion.js @@ -1,4 +1,4 @@ -//   Copyright (C) 2018 +//   Copyright (C) 2019  //   Matthias P. Braendli, matthias.braendli@mpb.li  //  //    http://www.opendigitalradio.org @@ -18,6 +18,8 @@  //   You should have received a copy of the GNU General Public License  //   along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +var adapt_dumps = []; +  function resultrefresh() {      var jqxhr = doApiRequestGET("/api/dpd_results", function(data) {          var summary = ""; @@ -29,6 +31,30 @@ function resultrefresh() {          $('#dpdresults').html(summary);          $('#dpdstatus').text(data['state']); +        var percentage = data['stateprogress']; +        if (percentage > 100) { +            percentage = 100; +        } +        $('#dpdprogress').css('width', percentage + '%'); +        $('#dpdprogresstext').text(percentage + '%'); + +        if (data['statplot']) { +            $('#dpdcapturestats').attr('src', data['statplot']); +        } +        else { +            $('#dpdcapturestats').attr('src', ""); +        } + +        $('#dpdmodeldata').html(data['modeldata']); + +        if (data['modelplot']) { +            $('#dpdmodelplot').attr('src', data['modelplot']); +        } +        else { +            $('#dpdmodelplot').attr('src', ""); +        } + +        adapt_dumps = data['adapt_dumps'];      });      jqxhr.always(function() { @@ -36,8 +62,32 @@ function resultrefresh() {      });  } +function adaptdumpsrefresh() { +    $('#dpdadaptdumps').html(""); + +    $.each(adapt_dumps, function(i, item) { +        console.log(item); + +        if (isNaN(+item)) { +            $('#dpdadaptdumps').append($('<option>', { +                value: item, +                text : "DPD settings from " + item, +            })); +        } +        else { +            var d = new Date(0); +            d.setUTCSeconds(item); + +            $('#dpdadaptdumps').append($('<option>', { +                value: item, +                text : "DPD settings from " + d.toISOString(), +            })); +        } +    }); +} +  $(function(){ -    setTimeout(resultrefresh, 2000); +    setTimeout(resultrefresh, 20);      $('#calibratebtn').click(function() {          doApiRequestPOST("/api/dpd_calibrate", {}, function(data) { @@ -45,72 +95,39 @@ $(function(){          });      }); -}); - -/* -function calibraterefresh() { -    doApiRequestGET("/api/calibrate", function(data) { -        var text = "Captured TX signal and feedback." + -            " TX median: " + data['tx_median'] + -            " RX median: " + data['rx_median'] + -            " with relative timestamp offset " + -            (data['tx_ts'] - data['rx_ts']) + -            " and measured offset " + data['coarse_offset'] + -            ". Correlation: " + data['correlation']; -        $('#calibrationresults').text(text); +    $('#triggerbtn').click(function() { +        doApiRequestPOST("/api/dpd_trigger_run", {}, function(data) { +            console.log("run succeeded: " + JSON.stringify(data)); +        });      }); -} -$(function(){ -    $('#refreshframesbtn').click(function() { -        var d = new Date(); -        var n = d.getTime(); -        $('#txframeimg').src = "dpd/txframe.png?cachebreak=" + n; -        $('#rxframeimg').src = "dpd/rxframe.png?cachebreak=" + n; +    $('#adaptbtn').click(function() { +        doApiRequestPOST("/api/dpd_adapt", {}, function(data) { +            console.log("adapt succeeded: " + JSON.stringify(data)); +        });      }); -    $('#capturebutton').click(function() { -        doApiRequestPOST("/api/trigger_capture", {}, function(data) { -            console.log("trigger_capture succeeded: " + JSON.stringify(data)); + +    $('#resetbtn').click(function() { +        doApiRequestPOST("/api/dpd_reset", {}, function(data) { +            console.log("reset succeeded: " + JSON.stringify(data));          });      }); -    $('#dpdstatusbutton').click(function() { -        doApiRequestGET("/api/dpd_status", function(data) { -            console.log("dpd_status succeeded: " + JSON.stringify(data)); -            $('#histogram').text(data.histogram); -            $('#capturestatus').text(data.capture.status); -            $('#capturelength').text(data.capture.length); -            $('#tx_median').text(data.capture.tx_median); -            $('#rx_median').text(data.capture.rx_median); -        }); +    $('#adaptdumpsrefreshbtn').click(adaptdumpsrefresh); + +    $('#adaptdumpsload').click(function() { +        var elt = document.getElementById("dpdadaptdumps"); -    $.ajax({ -        type: "GET", -        url: "/api/dpd_capture_pointcloud", - -        error: function(data) { -            if (data.status == 500) { -                var errorWindow = window.open("", "_self"); -                errorWindow.document.write(data.responseText); -            } -            else { -                $.gritter.add({ title: 'API', -                    text: "AJAX failed: " + data.statusText, -                    image: '/fonts/warning.png', -                    sticky: true, -                }); -            } -        }, -        success: function(data) { -            $('#dpd_pointcloud').value(data) +        if (elt.selectedIndex != -1) { +            var selectedoption = elt.options[elt.selectedIndex].value; +            doApiRequestPOST("/api/dpd_restore_dump", {dump_id: selectedoption}, function(data) { +                console.log("reset succeeded: " + JSON.stringify(data)); +            });          } -    })      });  }); -*/ -  // ToolTip init  $(function(){ diff --git a/python/gui/static/js/odr-rcvalues.js b/python/gui/static/js/odr-rcvalues.js index f49674c..486a8a7 100644 --- a/python/gui/static/js/odr-rcvalues.js +++ b/python/gui/static/js/odr-rcvalues.js @@ -31,22 +31,28 @@ function requestStatus() {      doApiRequestGET("/api/rc_parameters", function(data) {          console.log(data); -        let keys = Object.keys(data); -        keys.sort(); +        let controllable_names = Object.keys(data); +        controllable_names.sort();          var key1; -        for (key1 in keys) { -            let keys2 = Object.keys(data[keys[key1]]); -            keys2.sort(); +        for (key1 in controllable_names) { +            let param_names = Object.keys(data[controllable_names[key1]]); +            param_names.sort();              var key2; -            for (key2 in keys2) { -                var param = data[keys[key1]][keys2[key2]]; -                var key = keys[key1] + "_" + keys2[key2]; +            for (key2 in param_names) { +                var name_controllable = controllable_names[key1]; +                var name_param = param_names[key2]; +                var key = name_controllable + "_" + name_param; + +                var param = data[name_controllable][name_param];                  var valueentry = '<input type="text" id="input'+key+'" ' +                      'value="' + param['value'] + '">' +                      '<button type="button" class="btn btn-xs btn-warning"' + -                    'id="button'+key+'" >upd</button>'; +                    'id="button'+key+'" ' + +                    'data-controllable="'+name_controllable+'" ' + +                    'data-param="'+name_param+'" ' + +                    '>upd</button>';                  $('#rctable > tbody:last').append(                      '<tr><td>'+key+'</td>'+ @@ -54,7 +60,10 @@ function requestStatus() {                      '<td>'+param['help']+'</td></tr>');                  $('#button'+key).click(function() { -                    buttonSetRc("input"+key, key1, key2); +                    var attr_c = this.getAttribute('data-controllable'); +                    var attr_p = this.getAttribute('data-param'); +                    var k = attr_c + "_" + attr_p; +                    buttonSetRc("input"+k, attr_c, attr_p);                  });              }          } diff --git a/python/gui/templates/about.html b/python/gui/templates/about.html index 3a05230..b781d54 100644 --- a/python/gui/templates/about.html +++ b/python/gui/templates/about.html @@ -1,24 +1,3 @@ -<!-- -   Copyright (C) 2018 -   Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. ---> - -  <!DOCTYPE html>  <html lang="en"> @@ -63,3 +42,24 @@ along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.  </html> +<!-- +   Copyright (C) 2018 +   Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +--> + + diff --git a/python/gui/templates/home.html b/python/gui/templates/home.html index 5cb29f8..398df37 100644 --- a/python/gui/templates/home.html +++ b/python/gui/templates/home.html @@ -1,3 +1,59 @@ +<!DOCTYPE html> +<html lang="en"> + +{% include 'head.html' %} + +<body> +    {% include 'body-nav.html' %} + +    <div class="container-fluid"> +        <div class="jumbotron"> +            <h1>Opendigitalradio</h1><h2>ODR-DabMod Status Check +                <span id="overall_state" class="glyphicon glyphicon-refresh glyphicon-refresh-animate"></span> +            </h2> +            <div class="well well-sm"> +                <p>ODR-DabMod +                </p> +                <ul> +                    <li>Answering to RC: +                        <span id="is_rc_ok" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_rc_ok_comment"><span> +                    </li> +                    <li>Frame generation: +                        <span id="is_modulating" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_modulating_comment"><span> +                    </li> +                    <li>GPSDO status: +                        <span id="is_gpsdo_ok" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_gpsdo_ok_comment"><span> +                    </li> +                    <li>Underruns: +                        <span id="is_underrunning" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_underrunning_comment"><span> +                    </li> +                    <li>Late packets: +                        <span id="is_late" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_late_comment"><span> +                    </li> +                </ul> + +                <p>Checking predistortion +                <ul> +                    <li>Sample rate at 4x native rate: +                        <span id="is_rate_4x" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_rate_4x_comment"><span> +                    </li> +                    <li>DPDCE running: +                        <span id="is_dpdce_running" class="glyphicon glyphicon-question-sign"></span> +                        <span id="is_dpdce_running_comment"><span> +                    </li> +                </ul> +            </div> +        </div> +    </div> +</body> +</html> +  <!--     Copyright (C) 2018     Matthias P. Braendli, matthias.braendli@mpb.li @@ -17,19 +73,3 @@ GNU General Public License for more details.  You should have received a copy of the GNU General Public License  along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.  --> - -<!DOCTYPE html> -<html lang="en"> - -{% include 'head.html' %} - -<body> -	{% include 'body-nav.html' %} - -	<div class="container-fluid"> -		<div class="jumbotron"> -            <h1>Opendigitalradio</h1><h2>ODR-DabMod Interface</h2> -		</div> -	</div> -</body> -</html> diff --git a/python/gui/templates/modulator.html b/python/gui/templates/modulator.html index 6deffb1..016344a 100644 --- a/python/gui/templates/modulator.html +++ b/python/gui/templates/modulator.html @@ -1,23 +1,3 @@ -<!-- -   Copyright (C) 2018 -   Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. ---> -  <!DOCTYPE html>  <html lang="en"> @@ -71,3 +51,23 @@ along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.    </div>  </body>  </html> + +<!-- +   Copyright (C) 2018 +   Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/gui/templates/predistortion.html b/python/gui/templates/predistortion.html index cc5ecb0..62e8503 100644 --- a/python/gui/templates/predistortion.html +++ b/python/gui/templates/predistortion.html @@ -1,23 +1,3 @@ -<!-- -   Copyright (C) 2018 -   Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. ---> -  <!DOCTYPE html>  <html lang="en"> @@ -31,42 +11,87 @@ along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.        <div class="panel panel-default">          <div class="panel-heading">Status and calibration</div>          <div class="panel-body"> -          <div>Current DPDCE status: <span id="dpdstatus" style="font-weight:bold;">N/A</span> -            <div class="well well-sm" id="dpdresults">N/A</div> + +          <div class="container-fluid"> +            <div class="row"> +              <div class="col-sm-4"> +                <h2>Current DPDCE status</h2> +                <div> +                  <div id="dpdstatus" style="font-weight:bold;">N/A</div> +                  <div class="progress"> +                    <div id="dpdprogress" class="progress-bar" role="progressbar" style="width:0%"> +                      <span id="dpdprogresstext"></span> +                    </div> +                  </div> +                </div> +                <div class="well well-sm" id="dpdresults">N/A</div> +              </div> +              <div class="col-sm-4"> +                <h2>List of saved DPD settings</h2> +                <!--TODO: 'erase' and 'clear' buttons. Show DPD settings in tooltip?--> +                <p>This list contains previously used predistortion settings that you +                can recall.</p> +                <p> +                <select id="dpdadaptdumps" size="8" style="width:70%" multiple></select> +                </p> +                <p> +                <button type="button" class="btn btn-sm btn-info" id="adaptdumpsrefreshbtn">Refresh +                </button> +                <button type="button" class="btn btn-sm btn-warning" id="adaptdumpsload">Load and Apply +                </button> +                </p> +              </div> +              <div class="col-sm-4"> +                <h2>Summary</h2> +                <p>Calibration needs to be done once before the PA model +                can be trained. Every time calibration is changed, the predistortion +                parameters are invalidated!</p> +                <p>Once calibration succeeded and correct RX gain is set, you +                can trigger a capture and model the PA. Usually, several capture +                runs are needed before the model can be trained.</p> +                <p>The capture and model analysis will calculate a new set of +                DPD model data, that you can apply using the Update Predistorter button.</p> +                <p>The reset button allows you to reset the computation engine. It does not +                modify the currently active predistorter.</p> +              </div> +            </div>            </div> -          <div>Calibration needs to be done once before the PA model -          can be trained. Every time calibration is changed, the predistortion -          parameters are invalidated!</div>            <button type="button" class="btn btn-sm btn-warning" id="calibratebtn">              Calibrate</button> +          <button type="button" class="btn btn-sm btn-warning" id="triggerbtn"> +            Trigger Capture and PA Modeling</button> +          <button type="button" class="btn btn-sm btn-warning" id="adaptbtn"> +            Update Predistorter</button> +          <button type="button" class="btn btn-sm btn-info" id="resetbtn"> +            Reset Capture and Model</button>          </div>        </div> -      <!-- +        <div class="panel panel-default"> -        <div class="panel-heading">Capture TX and RX frames</div> +        <div class="panel-heading">Capture Statistics</div>          <div class="panel-body"> -          <div> -            <img id="txframeimg" src="dpd/txframe.png" width="320" height="240" /> -            <img id="rxframeimg" src="dpd/rxframe.png" width="320" height="240" /> -          </div> -          <div> -            <button type="button" class="btn btn-sm btn-info" id="refreshframesbtn"> -              Refresh</button> -          </div> +          <img id="dpdcapturestats" />          </div>        </div> -        <div class="panel panel-default"> -        <div class="panel-heading">Capture</div> +        <div class="panel-heading">AM/AM and AM/PM Model</div>          <div class="panel-body"> -          <div>On pressing this button, -            the DPDCE will trigger a capture and a quick data -            analysis, without updating any DPD models.</div> -          <button type="button" class="btn btn-sm btn-info" id="capturebutton"> -            Capture</button> +          <div class="container-fluid"> +            <div class="row"> +              <div class="col-sm-2"> +                <p>Model data:</p> +              </div> +              <div class="col-sm-10"> +                <pre id="dpdmodeldata"></pre> +              </div> +            </div> +          </div> +          <img id="dpdmodelplot" />          </div>        </div> + +      <!--        <div class="panel panel-default">          <div class="panel-heading">Status</div>          <div class="panel-body"> @@ -87,3 +112,23 @@ along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.    </div>  </body>  </html> + +<!-- +   Copyright (C) 2019 +   Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/gui/templates/rcvalues.html b/python/gui/templates/rcvalues.html index c1786bc..9e607bc 100644 --- a/python/gui/templates/rcvalues.html +++ b/python/gui/templates/rcvalues.html @@ -1,23 +1,3 @@ -<!-- -   Copyright (C) 2018 -   Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. ---> -  <!DOCTYPE html>  <html lang="en"> @@ -46,3 +26,22 @@ along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.      </div>  </body>  </html> +<!-- +   Copyright (C) 2018 +   Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/lib/yamlrpc.py b/python/lib/yamlrpc.py index d963601..67c65ff 100644 --- a/python/lib/yamlrpc.py +++ b/python/lib/yamlrpc.py @@ -23,6 +23,9 @@  """yamlrpc is json-rpc, except that it's yaml and not json.""" +# This maybe won't work over ethernet, but for localhost it's ok +UDP_PACKETSIZE = 2048 +  # Same as jsonrpc version we're aiming to mirror in YAML  YAMLRPC_VERSION = "2.0" @@ -80,9 +83,9 @@ class Socket:      def receive_response(self, expected_msg_id: int):          try: -            data, addr = self.socket.recvfrom(512) +            data, addr = self.socket.recvfrom(UDP_PACKETSIZE)          except socket.timeout as to: -            raise TimeoutError("Timeout: " + str(to)) +            raise TimeoutError()          y = yaml.load(data.decode()) @@ -117,7 +120,7 @@ class Socket:      def receive_request(self):          try: -            data, addr = self.socket.recvfrom(512) +            data, addr = self.socket.recvfrom(UDP_PACKETSIZE)          except socket.timeout as to:              raise TimeoutError("Timeout: " + str(to)) diff --git a/python/lib/zmqrc.py b/python/lib/zmqrc.py index 3897d7a..423f91d 100644 --- a/python/lib/zmqrc.py +++ b/python/lib/zmqrc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3  # -*- coding: utf-8 -*-  #  #   Copyright (C) 2018 @@ -22,15 +22,16 @@  #   along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.  import zmq  import json +from typing import List -class ModRemoteControl(object): +class ModRemoteControl:      """Interact with ODR-DabMod using the ZMQ RC"""      def __init__(self, mod_host, mod_port=9400):          self._host = mod_host          self._port = mod_port          self._ctx = zmq.Context() -    def _read(self, message_parts): +    def _read(self, message_parts: List[str]):          sock = zmq.Socket(self._ctx, zmq.REQ)          sock.setsockopt(zmq.LINGER, 0)          sock.connect("tcp://{}:{}".format(self._host, self._port)) @@ -70,14 +71,14 @@ class ModRemoteControl(object):          return modules -    def get_param_value(self, module, param): +    def get_param_value(self, module: str, param: str) -> str:          value = self._read(['get', module, param])          if value[0] == 'fail':              raise ValueError("Error getting param: {}".format(value[1]))          else:              return value[0] -    def set_param_value(self, module, param, value): +    def set_param_value(self, module: str, param: str, value: str) -> None:          ret = self._read(['set', module, param, value])          if ret[0] == 'fail':              raise ValueError("Error setting param: {}".format(ret[1]))  | 
