diff options
author | Matthias P. Braendli <matthias.braendli@mpb.li> | 2019-01-23 11:00:02 +0100 |
---|---|---|
committer | Matthias P. Braendli <matthias.braendli@mpb.li> | 2019-01-23 11:00:02 +0100 |
commit | 201d711a1d3dfbe46d622871731005937598e790 (patch) | |
tree | e43a95ee027e1be6ca8621f9e2c78aaf932a3421 /python | |
parent | 674228bedb325384f12602350ab36d075b5509a3 (diff) | |
parent | e0abfc3728fb56519fa2507d2468214e2a633c98 (diff) | |
download | dabmod-201d711a1d3dfbe46d622871731005937598e790.tar.gz dabmod-201d711a1d3dfbe46d622871731005937598e790.tar.bz2 dabmod-201d711a1d3dfbe46d622871731005937598e790.zip |
Merge branch 'next' into lime
Diffstat (limited to 'python')
-rw-r--r-- | python/dpd/Adapt.py | 291 | ||||
-rw-r--r-- | python/dpd/ExtractStatistic.py | 37 | ||||
-rw-r--r-- | python/dpd/GlobalConfig.py | 11 | ||||
-rw-r--r-- | python/dpd/Measure.py | 36 | ||||
-rw-r--r-- | python/dpd/Model_AM.py | 122 | ||||
-rw-r--r-- | python/dpd/Model_PM.py | 124 | ||||
-rw-r--r-- | python/dpd/Model_Poly.py | 147 | ||||
-rw-r--r-- | python/dpd/RX_Agc.py | 39 | ||||
-rwxr-xr-x | python/dpdce.py | 304 | ||||
-rw-r--r-- | python/gui-dpdce.ini | 5 | ||||
-rwxr-xr-x | python/gui.py | 6 | ||||
-rwxr-xr-x | python/gui/api.py | 47 | ||||
-rw-r--r-- | python/gui/static/css/odr.css | 14 | ||||
-rw-r--r-- | python/gui/static/js/odr-home.js | 240 | ||||
-rw-r--r-- | python/gui/static/js/odr-predistortion.js | 129 | ||||
-rw-r--r-- | python/gui/static/js/odr-rcvalues.js | 29 | ||||
-rw-r--r-- | python/gui/templates/about.html | 42 | ||||
-rw-r--r-- | python/gui/templates/home.html | 72 | ||||
-rw-r--r-- | python/gui/templates/modulator.html | 40 | ||||
-rw-r--r-- | python/gui/templates/predistortion.html | 129 | ||||
-rw-r--r-- | python/gui/templates/rcvalues.html | 39 | ||||
-rw-r--r-- | python/lib/yamlrpc.py | 9 | ||||
-rw-r--r-- | python/lib/zmqrc.py | 11 |
23 files changed, 1180 insertions, 743 deletions
diff --git a/python/dpd/Adapt.py b/python/dpd/Adapt.py index 840aee9..a30f0c8 100644 --- a/python/dpd/Adapt.py +++ b/python/dpd/Adapt.py @@ -9,263 +9,184 @@ This module is used to change settings of ODR-DabMod using the ZMQ remote control socket. """ -import zmq import logging import numpy as np -import os -import datetime +import os.path import pickle +from lib import zmqrc +from typing import List LUT_LEN = 32 FORMAT_POLY = 1 FORMAT_LUT = 2 -def _write_poly_coef_file(coefs_am, coefs_pm, path): - assert (len(coefs_am) == len(coefs_pm)) +def _write_poly_coef_file(coefs_am: List[float], coefs_pm: List[float], path: str) -> None: + assert len(coefs_am) == len(coefs_pm) - f = open(path, 'w') - f.write("{}\n{}\n".format(FORMAT_POLY, len(coefs_am))) - for coef in coefs_am: - f.write("{}\n".format(coef)) - for coef in coefs_pm: - f.write("{}\n".format(coef)) - f.close() + with open(path, 'w') as f: + f.write("{}\n{}\n".format(FORMAT_POLY, len(coefs_am))) + for coef in coefs_am: + f.write("{}\n".format(coef)) + for coef in coefs_pm: + f.write("{}\n".format(coef)) -def _write_lut_file(scalefactor, lut, path): - assert (len(lut) == LUT_LEN) +def _write_lut_file(scalefactor: float, lut: List[complex], path: str) -> None: + assert len(lut) == LUT_LEN - f = open(path, 'w') - f.write("{}\n{}\n".format(FORMAT_LUT, scalefactor)) - for coef in lut: - f.write("{}\n{}\n".format(coef.real, coef.imag)) - f.close() + with open(path, 'w') as f: + f.write("{}\n{}\n".format(FORMAT_LUT, scalefactor)) + for coef in lut: + f.write("{}\n{}\n".format(coef.real, coef.imag)) -def dpddata_to_str(dpddata): +def dpddata_to_str(dpddata) -> str: if dpddata[0] == "poly": coefs_am = dpddata[1] coefs_pm = dpddata[2] - return "dpd_coefs_am {}, dpd_coefs_pm {}".format( + return "Poly: AM/AM {}, AM/PM {}".format( coefs_am, coefs_pm) elif dpddata[0] == "lut": scalefactor = dpddata[1] lut = dpddata[2] - return "LUT scalefactor {}, LUT {}".format( + return "LUT: scalefactor {}, LUT {}".format( scalefactor, lut) else: raise ValueError("Unknown dpddata type {}".format(dpddata[0])) class Adapt: - """Uses the ZMQ remote control to change parameters of the DabMod + """Uses the ZMQ remote control to change parameters of the DabMod """ - Parameters - ---------- - port : int - Port at which the ODR-DabMod is listening to connect the - ZMQ remote control. - """ - - def __init__(self, port, coef_path, plot_location): + def __init__(self, port: int, coef_path: str, plot_location: str): logging.debug("Instantiate Adapt object") - self.port = port - self.coef_path = coef_path - self.plot_location = plot_location - self.host = "localhost" - self._context = zmq.Context() - - def _connect(self): - """Establish the connection to ODR-DabMod using - a ZMQ socket that is in request mode (Client). - Returns a socket""" - sock = self._context.socket(zmq.REQ) - poller = zmq.Poller() - poller.register(sock, zmq.POLLIN) - - sock.connect("tcp://%s:%d" % (self.host, self.port)) - - sock.send(b"ping") - - socks = dict(poller.poll(1000)) - if socks: - if socks.get(sock) == zmq.POLLIN: - data = [el.decode() for el in sock.recv_multipart()] - - if data != ['ok']: - raise RuntimeError( - "Invalid ZMQ RC answer to 'ping' at %s %d: %s" % - (self.host, self.port, data)) - else: - sock.close(linger=10) - raise RuntimeError( - "ZMQ RC does not respond to 'ping' at %s %d" % - (self.host, self.port)) - - return sock - - def send_receive(self, message): - """Send a message to ODR-DabMod. It always - returns the answer ODR-DabMod sends back. + self._port = port + self._coef_path = coef_path + self._plot_location = plot_location + self._host = "localhost" + self._mod_rc = zmqrc.ModRemoteControl(self._host, self._port) - An example message could be - "get sdr txgain" or "set sdr txgain 50" - - Parameter - --------- - message : str - The message string that will be sent to the receiver. - """ - sock = self._connect() - logging.debug("Send message: %s" % message) - msg_parts = message.split(" ") - for i, part in enumerate(msg_parts): - if i == len(msg_parts) - 1: - f = 0 - else: - f = zmq.SNDMORE - - sock.send(part.encode(), flags=f) - - data = [el.decode() for el in sock.recv_multipart()] - logging.debug("Received message: %s" % message) - return data - - def set_txgain(self, gain): - """Set a new txgain for the ODR-DabMod. - - Parameters - ---------- - gain : int - new TX gain, in the same format as ODR-DabMod's config file - """ + def set_txgain(self, gain : float) -> None: # TODO this is specific to the B200 if gain < 0 or gain > 89: raise ValueError("Gain has to be in [0,89]") - return self.send_receive("set sdr txgain %.4f" % float(gain)) + self._mod_rc.set_param_value("sdr", "txgain", "%.4f" % float(gain)) - def get_txgain(self): - """Get the txgain value in dB for the ODR-DabMod.""" - # TODO handle failure - return float(self.send_receive("get sdr txgain")[0]) + def get_txgain(self) -> float: + """Get the txgain value in dB, or -1 in case of error""" + try: + return float(self._mod_rc.get_param_value("sdr", "txgain")) + except ValueError as e: + logging.warning(f"Adapt: get_txgain error: {e}") + return -1.0 - def set_rxgain(self, gain): - """Set a new rxgain for the ODR-DabMod. - - Parameters - ---------- - gain : int - new RX gain, in the same format as ODR-DabMod's config file - """ + def set_rxgain(self, gain: float) -> None: # TODO this is specific to the B200 if gain < 0 or gain > 89: raise ValueError("Gain has to be in [0,89]") - return self.send_receive("set sdr rxgain %.4f" % float(gain)) - - def get_rxgain(self): - """Get the rxgain value in dB for the ODR-DabMod.""" - # TODO handle failure - return float(self.send_receive("get sdr rxgain")[0]) - - def set_digital_gain(self, gain): - """Set a new rxgain for the ODR-DabMod. - - Parameters - ---------- - gain : int - new RX gain, in the same format as ODR-DabMod's config file - """ - msg = "set gain digital %.5f" % gain - return self.send_receive(msg) - - def get_digital_gain(self): - """Get the rxgain value in dB for the ODR-DabMod.""" - # TODO handle failure - return float(self.send_receive("get gain digital")[0]) + self._mod_rc.set_param_value("sdr", "rxgain", "%.4f" % float(gain)) + + def get_rxgain(self) -> float: + """Get the rxgain value in dB, or -1 in case of error""" + try: + return float(self._mod_rc.get_param_value("sdr", "rxgain")) + except ValueError as e: + logging.warning(f"Adapt: get_rxgain error: {e}") + return -1.0 + + def set_digital_gain(self, gain: float) -> None: + self._mod_rc.set_param_value("gain", "digital", "%.5f" % float(gain)) + + def get_digital_gain(self) -> float: + """Get the digital gain value in linear scale, or -1 in case + of error""" + try: + return float(self._mod_rc.get_param_value("gain", "digital")) + except ValueError as e: + logging.warning(f"Adapt: get_digital_gain error: {e}") + return -1.0 def get_predistorter(self): """Load the coefficients from the file in the format given in the README, return ("poly", [AM coef], [PM coef]) or ("lut", scalefactor, [LUT entries]) """ - f = open(self.coef_path, 'r') - lines = f.readlines() - predistorter_format = int(lines[0]) - if predistorter_format == FORMAT_POLY: - coefs_am_out = [] - coefs_pm_out = [] - n_coefs = int(lines[1]) - coefs = [float(l) for l in lines[2:]] - i = 0 - for c in coefs: - if i < n_coefs: - coefs_am_out.append(c) - elif i < 2 * n_coefs: - coefs_pm_out.append(c) - else: - raise ValueError( - 'Incorrect coef file format: too many' - ' coefficients in {}, should be {}, coefs are {}' - .format(self.coef_path, n_coefs, coefs)) - i += 1 - f.close() - return 'poly', coefs_am_out, coefs_pm_out - elif predistorter_format == FORMAT_LUT: - scalefactor = int(lines[1]) - coefs = np.array([float(l) for l in lines[2:]], dtype=np.float32) - coefs = coefs.reshape((-1, 2)) - lut = coefs[..., 0] + 1j * coefs[..., 1] - if len(lut) != LUT_LEN: - raise ValueError("Incorrect number of LUT entries ({} expected {})".format(len(lut), LUT_LEN)) - return 'lut', scalefactor, lut - else: - raise ValueError("Unknown predistorter format {}".format(predistorter_format)) + with open(self._coef_path, 'r') as f: + lines = f.readlines() + predistorter_format = int(lines[0]) + if predistorter_format == FORMAT_POLY: + coefs_am_out = [] + coefs_pm_out = [] + n_coefs = int(lines[1]) + coefs = [float(l) for l in lines[2:]] + for i, c in enumerate(coefs): + if i < n_coefs: + coefs_am_out.append(c) + elif i < 2 * n_coefs: + coefs_pm_out.append(c) + else: + raise ValueError( + 'Incorrect coef file format: too many' + ' coefficients in {}, should be {}, coefs are {}' + .format(self._coef_path, n_coefs, coefs)) + return 'poly', coefs_am_out, coefs_pm_out + elif predistorter_format == FORMAT_LUT: + scalefactor = int(lines[1]) + coefs = np.array([float(l) for l in lines[2:]], dtype=np.float32) + coefs = coefs.reshape((-1, 2)) + lut = coefs[..., 0] + 1j * coefs[..., 1] + if len(lut) != LUT_LEN: + raise ValueError("Incorrect number of LUT entries ({} expected {})".format(len(lut), LUT_LEN)) + return 'lut', scalefactor, lut + else: + raise ValueError("Unknown predistorter format {}".format(predistorter_format)) - def set_predistorter(self, dpddata): + def set_predistorter(self, dpddata) -> None: """Update the predistorter data in the modulator. Takes the same tuple format as argument than the one returned get_predistorter()""" if dpddata[0] == "poly": coefs_am = dpddata[1] coefs_pm = dpddata[2] - _write_poly_coef_file(coefs_am, coefs_pm, self.coef_path) + _write_poly_coef_file(coefs_am, coefs_pm, self._coef_path) elif dpddata[0] == "lut": scalefactor = dpddata[1] lut = dpddata[2] - _write_lut_file(scalefactor, lut, self.coef_path) + _write_lut_file(scalefactor, lut, self._coef_path) else: raise ValueError("Unknown predistorter '{}'".format(dpddata[0])) - self.send_receive("set memlesspoly coeffile {}".format(self.coef_path)) + self._mod_rc.set_param_value("memlesspoly", "coeffile", self._coef_path) - def dump(self, path=None): + def dump(self, path: str) -> None: """Backup current settings to a file""" - dt = datetime.datetime.now().isoformat() - if path is None: - if self.plot_location is not None: - path = self.plot_location + "/" + dt + "_adapt.pkl" - else: - raise Exception("Cannot dump Adapt without either plot_location or path set") + d = { "txgain": self.get_txgain(), "rxgain": self.get_rxgain(), "digital_gain": self.get_digital_gain(), - "predistorter": self.get_predistorter() + "dpddata": self.get_predistorter() } + with open(path, "wb") as f: pickle.dump(d, f) - return path - - def load(self, path): + def restore(self, path: str): """Restore settings from a file""" with open(path, "rb") as f: d = pickle.load(f) - self.set_txgain(d["txgain"]) + self.set_txgain(0) + + # If any of the following fail, we will be running + # with the safe value of txgain=0 self.set_digital_gain(d["digital_gain"]) self.set_rxgain(d["rxgain"]) - self.set_predistorter(d["predistorter"]) + self.set_predistorter(d["dpddata"]) + self.set_txgain(d["txgain"]) + + return d # The MIT License (MIT) # -# Copyright (c) 2017 Andreas Steger, Matthias P. Braendli +# Copyright (c) 2019 Matthias P. Braendli +# Copyright (c) 2017 Andreas Steger # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/ExtractStatistic.py b/python/dpd/ExtractStatistic.py index 639513a..3518dba 100644 --- a/python/dpd/ExtractStatistic.py +++ b/python/dpd/ExtractStatistic.py @@ -38,14 +38,16 @@ class ExtractStatistic: """Calculate a low variance RX value for equally spaced tx values of a predefined range""" - def __init__(self, c): + def __init__(self, c, peak_amplitude): self.c = c + self._plot_data = None + # Number of measurements used to extract the statistic self.n_meas = 0 # Boundaries for the bins - self.tx_boundaries = np.linspace(c.ES_start, c.ES_end, c.ES_n_bins + 1) + self.tx_boundaries = np.linspace(0.0, peak_amplitude, c.ES_n_bins + 1) self.n_per_bin = c.ES_n_per_bin # List of rx values for each bin @@ -58,12 +60,14 @@ class ExtractStatistic: for i in range(c.ES_n_bins): self.tx_values_lists.append([]) - self.plot = c.ES_plot + def get_bin_info(self): + return "Binning: {} bins used for amplitudes between {} and {}".format( + len(self.tx_boundaries), np.min(self.tx_boundaries), np.max(self.tx_boundaries)) + + def plot(self, plot_path, title): + if self._plot_data is not None: + tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists = self._plot_data - def _plot_and_log(self, tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists): - if self.plot and self.c.plot_location is not None: - dt = datetime.datetime.now().isoformat() - fig_path = self.c.plot_location + "/" + dt + "_ExtractStatistic.png" sub_rows = 3 sub_cols = 1 fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) @@ -72,7 +76,7 @@ class ExtractStatistic: i_sub += 1 ax = plt.subplot(sub_rows, sub_cols, i_sub) ax.plot(tx_values, rx_values, - label="Estimated Values", + label="Averaged measurements", color="red") for i, tx_value in enumerate(tx_values): rx_values_list = self.rx_values_lists[i] @@ -80,17 +84,17 @@ class ExtractStatistic: np.abs(rx_values_list), s=0.1, color="black") - ax.set_title("Extracted Statistic") + ax.set_title("Extracted Statistic {}".format(title)) ax.set_xlabel("TX Amplitude") ax.set_ylabel("RX Amplitude") - ax.set_ylim(0, 0.8) - ax.set_xlim(0, 1.1) + ax.set_ylim(0, np.max(self.tx_boundaries)) # we expect a rougly a 1:1 correspondence between x and y + ax.set_xlim(0, np.max(self.tx_boundaries)) ax.legend(loc=4) i_sub += 1 ax = plt.subplot(sub_rows, sub_cols, i_sub) ax.plot(tx_values, np.rad2deg(phase_diffs_values), - label="Estimated Values", + label="Averaged measurements", color="red") for i, tx_value in enumerate(tx_values): phase_diff = phase_diffs_values_lists[i] @@ -101,7 +105,7 @@ class ExtractStatistic: ax.set_xlabel("TX Amplitude") ax.set_ylabel("Phase Difference") ax.set_ylim(-60, 60) - ax.set_xlim(0, 1.1) + ax.set_xlim(0, np.max(self.tx_boundaries)) ax.legend(loc=4) num = [] @@ -111,12 +115,12 @@ class ExtractStatistic: i_sub += 1 ax = plt.subplot(sub_rows, sub_cols, i_sub) ax.plot(num) - ax.set_xlabel("TX Amplitude") + ax.set_xlabel("TX Amplitude bin") ax.set_ylabel("Number of Samples") ax.set_ylim(0, self.n_per_bin * 1.2) fig.tight_layout() - fig.savefig(fig_path) + fig.savefig(plot_path) plt.close(fig) def _rx_value_per_bin(self): @@ -166,7 +170,7 @@ class ExtractStatistic: phase_diffs_values_lists = self._phase_diff_list_per_bin() phase_diffs_values = _phase_diff_value_per_bin(phase_diffs_values_lists) - self._plot_and_log(tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists) + self._plot_data = (tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists) tx_values_crop = np.array(tx_values, dtype=np.float32)[:idx_end] rx_values_crop = np.array(rx_values, dtype=np.float32)[:idx_end] @@ -176,6 +180,7 @@ class ExtractStatistic: # The MIT License (MIT) # # Copyright (c) 2017 Andreas Steger +# Copyright (c) 2018 Matthias P. Braendli # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/GlobalConfig.py b/python/dpd/GlobalConfig.py index 873b6ac..632a63b 100644 --- a/python/dpd/GlobalConfig.py +++ b/python/dpd/GlobalConfig.py @@ -10,7 +10,7 @@ import numpy as np class GlobalConfig: - def __init__(self, samplerate, plot_location: str): + def __init__(self, samplerate: int, plot_location: str): self.sample_rate = samplerate assert self.sample_rate == 8192000, "We only support constants for 8192000 sample rate: {}".format(self.sample_rate) @@ -26,6 +26,8 @@ class GlobalConfig: self.T_U = oversample * 2048 # Inverse of carrier spacing self.T_C = oversample * 504 # Duration of cyclic prefix + self.median_to_peak = 12 # Estimated value for a DAB OFDM signal + # Frequency Domain # example: np.delete(fft[3328:4865], 768) self.FFT_delta = 1536 # Number of carrier frequencies @@ -40,10 +42,8 @@ class GlobalConfig: self.phase_offset_per_sample = 1. / self.sample_rate * 2 * np.pi * 1000 # Constants for ExtractStatistic - self.ES_plot = plot - self.ES_start = 0.0 self.ES_end = 1.0 - self.ES_n_bins = 64 # Number of bins between ES_start and ES_end + self.ES_n_bins = 64 self.ES_n_per_bin = 128 # Number of measurements pre bin # Constants for Measure_Shoulder @@ -68,9 +68,6 @@ class GlobalConfig: # Constants for MER self.MER_plot = plot - # Constants for Model - self.MDL_plot = plot - # Constants for Model_PM # Set all phase offsets to zero for TX amplitude < MPM_tx_min self.MPM_tx_min = 0.1 diff --git a/python/dpd/Measure.py b/python/dpd/Measure.py index 489c4c0..e5a72c7 100644 --- a/python/dpd/Measure.py +++ b/python/dpd/Measure.py @@ -15,7 +15,7 @@ import logging class Measure: """Collect Measurement from DabMod""" - def __init__(self, config, samplerate, port, num_samples_to_request): + def __init__(self, config, samplerate : int, port : int, num_samples_to_request : int): logging.info("Instantiate Measure object") self.c = config self.samplerate = samplerate @@ -23,7 +23,7 @@ class Measure: self.port = port self.num_samples_to_request = num_samples_to_request - def _recv_exact(self, sock, num_bytes): + def _recv_exact(self, sock : socket.socket, num_bytes : int) -> bytes: """Receive an exact number of bytes from a socket. This is a wrapper around sock.recv() that can return less than the number of requested bytes. @@ -41,7 +41,7 @@ class Measure: bufs.append(b) return b''.join(bufs) - def receive_tcp(self): + def receive_tcp(self, num_samples_to_request : int): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(4) s.connect(('localhost', self.port)) @@ -49,8 +49,8 @@ class Measure: logging.debug("Send version") s.sendall(b"\x01") - logging.debug("Send request for {} samples".format(self.num_samples_to_request)) - s.sendall(struct.pack("=I", self.num_samples_to_request)) + logging.debug("Send request for {} samples".format(num_samples_to_request)) + s.sendall(struct.pack("=I", num_samples_to_request)) logging.debug("Wait for TX metadata") num_samps, tx_second, tx_pps = struct.unpack("=III", self._recv_exact(s, 12)) @@ -90,14 +90,35 @@ class Measure: return txframe, tx_ts, rxframe, rx_ts + def get_samples_unaligned(self, short=False): + """Connect to ODR-DabMod, retrieve TX and RX samples, load + into numpy arrays, and return a tuple + (txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median) + """ + + n_samps = int(self.num_samples_to_request / 4) if short else self.num_samples_to_request + txframe, tx_ts, rxframe, rx_ts = self.receive_tcp(n_samps) + + # Normalize received signal with sent signal + rx_median = np.median(np.abs(rxframe)) + tx_median = np.median(np.abs(txframe)) + rxframe = rxframe / rx_median * tx_median + + + logging.info( + "Measurement done, tx %d %s, rx %d %s" % + (len(txframe), txframe.dtype, len(rxframe), rxframe.dtype)) + + return txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median - def get_samples(self): + def get_samples(self, short=False): """Connect to ODR-DabMod, retrieve TX and RX samples, load into numpy arrays, and return a tuple (txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median) """ - txframe, tx_ts, rxframe, rx_ts = self.receive_tcp() + n_samps = int(self.num_samples_to_request / 4) if short else self.num_samples_to_request + txframe, tx_ts, rxframe, rx_ts = self.receive_tcp(n_samps) # Normalize received signal with sent signal rx_median = np.median(np.abs(rxframe)) @@ -116,6 +137,7 @@ class Measure: # The MIT License (MIT) # +# Copyright (c) 2018 Matthias P. Braendli # Copyright (c) 2017 Andreas Steger # # Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/python/dpd/Model_AM.py b/python/dpd/Model_AM.py deleted file mode 100644 index 75b226f..0000000 --- a/python/dpd/Model_AM.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- -# -# DPD Computation Engine, model implementation for Amplitude and not Phase -# -# http://www.opendigitalradio.org -# Licence: The MIT License, see notice at the end of this file - -import datetime -import os -import logging -import numpy as np -import matplotlib.pyplot as plt - - -def is_npfloat32(array): - assert isinstance(array, np.ndarray), type(array) - assert array.dtype == np.float32, array.dtype - assert array.flags.contiguous - assert not any(np.isnan(array)) - - -def check_input_get_next_coefs(tx_dpd, rx_received): - is_npfloat32(tx_dpd) - is_npfloat32(rx_received) - - -def poly(sig): - return np.array([sig ** i for i in range(1, 6)]).T - - -def fit_poly(tx_abs, rx_abs): - return np.linalg.lstsq(poly(rx_abs), tx_abs, rcond=None)[0] - - -def calc_line(coefs, min_amp, max_amp): - rx_range = np.linspace(min_amp, max_amp) - tx_est = np.sum(poly(rx_range) * coefs, axis=1) - return tx_est, rx_range - - -class Model_AM: - """Calculates new coefficients using the measurement and the previous - coefficients""" - - def __init__(self, - c, - learning_rate_am=1, - plot=False): - self.c = c - - self.learning_rate_am = learning_rate_am - self.plot = plot - - def _plot(self, tx_dpd, rx_received, coefs_am, coefs_am_new): - if self.plot and self.c.plot_location is not None: - tx_range, rx_est = calc_line(coefs_am, 0, 0.6) - tx_range_new, rx_est_new = calc_line(coefs_am_new, 0, 0.6) - - dt = datetime.datetime.now().isoformat() - fig_path = self.c.plot_location + "/" + dt + "_Model_AM.png" - sub_rows = 1 - sub_cols = 1 - fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) - i_sub = 0 - - i_sub += 1 - ax = plt.subplot(sub_rows, sub_cols, i_sub) - ax.plot(tx_range, rx_est, - label="Estimated TX", - alpha=0.3, - color="gray") - ax.plot(tx_range_new, rx_est_new, - label="New Estimated TX", - color="red") - ax.scatter(tx_dpd, rx_received, - label="Binned Data", - color="blue", - s=1) - ax.set_title("Model_AM") - ax.set_xlabel("TX Amplitude") - ax.set_ylabel("RX Amplitude") - ax.set_xlim(-0.5, 1.5) - ax.legend(loc=4) - - fig.tight_layout() - fig.savefig(fig_path) - plt.close(fig) - - def get_next_coefs(self, tx_dpd, rx_received, coefs_am): - """Calculate the next AM/AM coefficients using the extracted - statistic of TX and RX amplitude""" - check_input_get_next_coefs(tx_dpd, rx_received) - - coefs_am_new = fit_poly(tx_dpd, rx_received) - coefs_am_new = coefs_am + \ - self.learning_rate_am * (coefs_am_new - coefs_am) - - self._plot(tx_dpd, rx_received, coefs_am, coefs_am_new) - - return coefs_am_new - -# The MIT License (MIT) -# -# Copyright (c) 2017 Andreas Steger -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/python/dpd/Model_PM.py b/python/dpd/Model_PM.py deleted file mode 100644 index 7b80bf3..0000000 --- a/python/dpd/Model_PM.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- coding: utf-8 -*- -# -# DPD Computation Engine, model implementation for Amplitude and not Phase -# -# http://www.opendigitalradio.org -# Licence: The MIT License, see notice at the end of this file - -import datetime -import os -import logging -import numpy as np -import matplotlib.pyplot as plt - - -def is_npfloat32(array): - assert isinstance(array, np.ndarray), type(array) - assert array.dtype == np.float32, array.dtype - assert array.flags.contiguous - assert not any(np.isnan(array)) - - -def check_input_get_next_coefs(tx_dpd, phase_diff): - is_npfloat32(tx_dpd) - is_npfloat32(phase_diff) - - -class Model_PM: - """Calculates new coefficients using the measurement and the previous - coefficients""" - - def __init__(self, - c, - learning_rate_pm=1, - plot=False): - self.c = c - - self.learning_rate_pm = learning_rate_pm - self.plot = plot - - def _plot(self, tx_dpd, phase_diff, coefs_pm, coefs_pm_new): - if self.plot and self.c.plot_location is not None: - tx_range, phase_diff_est = self.calc_line(coefs_pm, 0, 0.6) - tx_range_new, phase_diff_est_new = self.calc_line(coefs_pm_new, 0, 0.6) - - dt = datetime.datetime.now().isoformat() - fig_path = self.c.plot_location + "/" + dt + "_Model_PM.png" - sub_rows = 1 - sub_cols = 1 - fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) - i_sub = 0 - - i_sub += 1 - ax = plt.subplot(sub_rows, sub_cols, i_sub) - ax.plot(tx_range, phase_diff_est, - label="Estimated Phase Diff", - alpha=0.3, - color="gray") - ax.plot(tx_range_new, phase_diff_est_new, - label="New Estimated Phase Diff", - color="red") - ax.scatter(tx_dpd, phase_diff, - label="Binned Data", - color="blue", - s=1) - ax.set_title("Model_PM") - ax.set_xlabel("TX Amplitude") - ax.set_ylabel("Phase DIff") - ax.legend(loc=4) - - fig.tight_layout() - fig.savefig(fig_path) - plt.close(fig) - - def _discard_small_values(self, tx_dpd, phase_diff): - """ Assumes that the phase for small tx amplitudes is zero""" - mask = tx_dpd < self.c.MPM_tx_min - phase_diff[mask] = 0 - return tx_dpd, phase_diff - - def poly(self, sig): - return np.array([sig ** i for i in range(0, 5)]).T - - def fit_poly(self, tx_abs, phase_diff): - return np.linalg.lstsq(self.poly(tx_abs), phase_diff, rcond=None)[0] - - def calc_line(self, coefs, min_amp, max_amp): - tx_range = np.linspace(min_amp, max_amp) - phase_diff = np.sum(self.poly(tx_range) * coefs, axis=1) - return tx_range, phase_diff - - def get_next_coefs(self, tx_dpd, phase_diff, coefs_pm): - """Calculate the next AM/PM coefficients using the extracted - statistic of TX amplitude and phase difference""" - tx_dpd, phase_diff = self._discard_small_values(tx_dpd, phase_diff) - check_input_get_next_coefs(tx_dpd, phase_diff) - - coefs_pm_new = self.fit_poly(tx_dpd, phase_diff) - - coefs_pm_new = coefs_pm + self.learning_rate_pm * (coefs_pm_new - coefs_pm) - self._plot(tx_dpd, phase_diff, coefs_pm, coefs_pm_new) - - return coefs_pm_new - -# The MIT License (MIT) -# -# Copyright (c) 2017 Andreas Steger -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/python/dpd/Model_Poly.py b/python/dpd/Model_Poly.py index c8f6135..ef3fed3 100644 --- a/python/dpd/Model_Poly.py +++ b/python/dpd/Model_Poly.py @@ -8,15 +8,13 @@ import os import logging import numpy as np +import matplotlib.pyplot as plt -import dpd.Model_AM as Model_AM -import dpd.Model_PM as Model_PM - - -def assert_np_float32(x): - assert isinstance(x, np.ndarray) - assert x.dtype == np.float32 - assert x.flags.contiguous +def assert_np_float32(array): + assert isinstance(array, np.ndarray), type(array) + assert array.dtype == np.float32, array.dtype + assert array.flags.contiguous + assert not any(np.isnan(array)) def _check_input_get_next_coefs(tx_abs, rx_abs, phase_diff): @@ -36,20 +34,73 @@ class Poly: """Calculates new coefficients using the measurement and the previous coefficients""" - def __init__(self, - c, - learning_rate_am=1.0, - learning_rate_pm=1.0): + def __init__(self, c, learning_rate_am=1.0, learning_rate_pm=1.0): self.c = c - self.plot = c.MDL_plot self.learning_rate_am = learning_rate_am self.learning_rate_pm = learning_rate_pm self.reset_coefs() - self.model_am = Model_AM.Model_AM(c, plot=self.plot) - self.model_pm = Model_PM.Model_PM(c, plot=self.plot) + def plot(self, plot_location, title): + if self._am_plot_data is not None and self._pm_plot_data is not None: + tx_dpd, rx_received, coefs_am, coefs_am_new = self._am_plot_data + + tx_range, rx_est = self._am_calc_line(coefs_am, 0, 0.6) + tx_range_new, rx_est_new = self._am_calc_line(coefs_am_new, 0, 0.6) + + sub_rows = 2 + sub_cols = 1 + fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6)) + i_sub = 0 + + # AM subplot + i_sub += 1 + ax = plt.subplot(sub_rows, sub_cols, i_sub) + ax.plot(tx_range, rx_est, + label="Estimated TX", + alpha=0.3, + color="gray") + ax.plot(tx_range_new, rx_est_new, + label="New Estimated TX", + color="red") + ax.scatter(tx_dpd, rx_received, + label="Binned Data", + color="blue", + s=1) + ax.set_title("Model AM and PM {}".format(title)) + ax.set_xlabel("TX Amplitude") + ax.set_ylabel("RX Amplitude") + ax.set_xlim(0, 1.0) + ax.legend(loc=4) + + # PM sub plot + tx_dpd, phase_diff, coefs_pm, coefs_pm_new = self._pm_plot_data + + tx_range, phase_diff_est = self._pm_calc_line(coefs_pm, 0, 0.6) + tx_range_new, phase_diff_est_new = self._pm_calc_line(coefs_pm_new, 0, 0.6) + + i_sub += 1 + ax = plt.subplot(sub_rows, sub_cols, i_sub) + ax.plot(tx_range, phase_diff_est, + label="Estimated Phase Diff", + alpha=0.3, + color="gray") + ax.plot(tx_range_new, phase_diff_est_new, + label="New Estimated Phase Diff", + color="red") + ax.scatter(tx_dpd, phase_diff, + label="Binned Data", + color="blue", + s=1) + ax.set_xlabel("TX Amplitude") + ax.set_ylabel("Phase DIff") + ax.set_xlim(0, 1.0) + ax.legend(loc=4) + + fig.tight_layout() + fig.savefig(plot_location) + plt.close(fig) def reset_coefs(self): self.coefs_am = np.zeros(5, dtype=np.float32) @@ -65,12 +116,8 @@ class Poly: """ _check_input_get_next_coefs(tx_abs, rx_abs, phase_diff) - if not lr is None: - self.model_am.learning_rate_am = lr - self.model_pm.learning_rate_pm = lr - - coefs_am_new = self.model_am.get_next_coefs(tx_abs, rx_abs, self.coefs_am) - coefs_pm_new = self.model_pm.get_next_coefs(tx_abs, phase_diff, self.coefs_pm) + coefs_am_new = self._am_get_next_coefs(tx_abs, rx_abs, self.coefs_am) + coefs_pm_new = self._pm_get_next_coefs(tx_abs, phase_diff, self.coefs_pm) self.coefs_am = self.coefs_am + (coefs_am_new - self.coefs_am) * self.learning_rate_am self.coefs_pm = self.coefs_pm + (coefs_pm_new - self.coefs_pm) * self.learning_rate_pm @@ -78,9 +125,67 @@ class Poly: def get_dpd_data(self): return "poly", self.coefs_am, self.coefs_pm + def set_dpd_data(self, dpddata): + if dpddata[0] != "poly" or len(dpddata) != 3: + raise ValueError("dpddata is not of 'poly' format") + _, self.coefs_am, self.coefs_pm = dpddata + + def _am_calc_line(self, coefs, min_amp, max_amp): + rx_range = np.linspace(min_amp, max_amp) + tx_est = np.sum(self._am_poly(rx_range) * coefs, axis=1) + return tx_est, rx_range + + def _am_poly(self, sig): + return np.array([sig ** i for i in range(1, 6)]).T + + def _am_fit_poly(self, tx_abs, rx_abs): + return np.linalg.lstsq(self._am_poly(rx_abs), tx_abs, rcond=None)[0] + + def _am_get_next_coefs(self, tx_dpd, rx_received, coefs_am): + """Calculate the next AM/AM coefficients using the extracted + statistic of TX and RX amplitude""" + + coefs_am_new = self._am_fit_poly(tx_dpd, rx_received) + coefs_am_new = coefs_am + \ + self.learning_rate_am * (coefs_am_new - coefs_am) + + self._am_plot_data = (tx_dpd, rx_received, coefs_am, coefs_am_new) + + return coefs_am_new + + def _pm_poly(self, sig): + return np.array([sig ** i for i in range(0, 5)]).T + + def _pm_calc_line(self, coefs, min_amp, max_amp): + tx_range = np.linspace(min_amp, max_amp) + phase_diff = np.sum(self._pm_poly(tx_range) * coefs, axis=1) + return tx_range, phase_diff + + def _discard_small_values(self, tx_dpd, phase_diff): + """ Assumes that the phase for small tx amplitudes is zero""" + mask = tx_dpd < self.c.MPM_tx_min + phase_diff[mask] = 0 + return tx_dpd, phase_diff + + def _pm_fit_poly(self, tx_abs, phase_diff): + return np.linalg.lstsq(self._pm_poly(tx_abs), phase_diff, rcond=None)[0] + + def _pm_get_next_coefs(self, tx_dpd, phase_diff, coefs_pm): + """Calculate the next AM/PM coefficients using the extracted + statistic of TX amplitude and phase difference""" + tx_dpd, phase_diff = self._discard_small_values(tx_dpd, phase_diff) + + coefs_pm_new = self._pm_fit_poly(tx_dpd, phase_diff) + + coefs_pm_new = coefs_pm + self.learning_rate_pm * (coefs_pm_new - coefs_pm) + self._pm_plot_data = (tx_dpd, phase_diff, coefs_pm, coefs_pm_new) + + return coefs_pm_new + # The MIT License (MIT) # # Copyright (c) 2017 Andreas Steger +# Copyright (c) 2018 Matthias P. Brandli # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/python/dpd/RX_Agc.py b/python/dpd/RX_Agc.py index 48ef7f3..bb940be 100644 --- a/python/dpd/RX_Agc.py +++ b/python/dpd/RX_Agc.py @@ -19,19 +19,19 @@ import dpd.Adapt as Adapt import dpd.Measure as Measure class Agc: - """The goal of the automatic gain control is to set the - RX gain to a value at which all received amplitudes can - be detected. This means that the maximum possible amplitude + """The goal of the automatic gain control is to set the + RX gain to a value at which all received amplitudes can + be detected. This means that the maximum possible amplitude should be quantized at the highest possible digital value. - A problem we have to face, is that the estimation of the - maximum amplitude by applying the max() function is very - unstable. This is due to the maximum’s rareness. Therefore - we estimate a far more robust value, such as the median, + A problem we have to face, is that the estimation of the + maximum amplitude by applying the max() function is very + unstable. This is due to the maximum’s rareness. Therefore + we estimate a far more robust value, such as the median, and then approximate the maximum amplitude from it. - Given this, we tune the RX gain in such a way, that the - received signal fulfills our desired property, of having + Given this, we tune the RX gain in such a way, that the + received signal fulfills our desired property, of having all amplitudes properly quantized.""" def __init__(self, measure, adapt, c): @@ -45,10 +45,15 @@ class Agc: self.peak_to_median = 1./c.RAGC_rx_median_target def run(self) -> Tuple[bool, str]: - self.adapt.set_rxgain(self.rxgain) + try: + self.adapt.set_rxgain(self.rxgain) + except ValueError as e: + return (False, "Setting RX gain to {} failed: {}".format(self.rxgain, e)) + time.sleep(0.5) + # Measure - txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = self.measure.get_samples() + txframe, tx_ts, rxframe, rx_ts, rx_median, tx_median = self.measure.get_samples_unaligned(short=False) # Estimate Maximum rx_peak = self.peak_to_median * rx_median @@ -68,11 +73,19 @@ class Agc: w = "Warning: calculated RX Gain={} is higher than maximum={}. RX feedback power should be increased.".format( self.rxgain, self.max_rxgain) logging.warning(w) + try: + # Reset to a low value, as we expect the user to reduce external attenuation + self.adapt.set_rxgain(30) + except ValueError as e: + return (False, "\n".join([measurements, w, "Setting RX gain to {} failed: {}".format(self.rxgain, e)])) return (False, "\n".join([measurements, w])) else: - self.adapt.set_rxgain(self.rxgain) + try: + self.adapt.set_rxgain(self.rxgain) + except ValueError as e: + return (False, "Setting RX gain to {} failed: {}".format(self.rxgain, e)) time.sleep(0.5) - return (True, measurements) + return (True, measurements) def plot_estimates(self): """Plots the estimate of for Max, Median, Mean for different diff --git a/python/dpdce.py b/python/dpdce.py index 838d265..cf98aa0 100755 --- a/python/dpdce.py +++ b/python/dpdce.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # DPD Computation Engine standalone main file. @@ -43,7 +43,8 @@ rc_port = config.getint('rc_port') samplerate = config.getint('samplerate') samps = config.getint('samps') coef_file = config['coef_file'] -log_folder = config['log_folder'] +logs_directory = config['logs_directory'] +plot_directory = config['plot_directory'] import logging import datetime @@ -52,7 +53,7 @@ save_logs = False # Simple usage scenarios don't need to clutter /tmp if save_logs: - dt = datetime.datetime.now().isoformat() + dt = datetime.datetime.utcnow().isoformat() logging_path = '/tmp/dpd_{}'.format(dt).replace('.', '_').replace(':', '-') print("Logs and plots written to {}".format(logging_path)) os.makedirs(logging_path) @@ -71,7 +72,7 @@ if save_logs: # add the handler to the root logger logging.getLogger('').addHandler(console) else: - dt = datetime.datetime.now().isoformat() + dt = datetime.datetime.utcnow().isoformat() logging.basicConfig(format='%(asctime)s - %(module)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO) @@ -79,10 +80,14 @@ else: logging.info("DPDCE starting up"); +import time import socket from lib import yamlrpc import numpy as np import traceback +import os.path +import glob +import re from threading import Thread, Lock from queue import Queue from dpd.Model import Poly @@ -96,13 +101,15 @@ from dpd.GlobalConfig import GlobalConfig from dpd.MER import MER from dpd.Measure_Shoulders import Measure_Shoulders -c = GlobalConfig(samplerate, logging_path) +plot_path = os.path.realpath(plot_directory) +coef_file = os.path.realpath(config['coef_file']) + +c = GlobalConfig(samplerate, plot_path) symbol_align = Symbol_align(c) mer = MER(c) meas_shoulders = Measure_Shoulders(c) meas = Measure(c, samplerate, dpd_port, samps) -extStat = ExtractStatistic(c) -adapt = Adapt(rc_port, coef_file, logging_path) +adapt = Adapt(rc_port, coef_file, plot_path) model = Poly(c) @@ -128,54 +135,271 @@ if cli_args.reset: cmd_socket = yamlrpc.Socket(bind_port=control_port) # The following is accessed by both threads and need to be locked -settings = { - 'rx_gain': rx_gain, - 'tx_gain': tx_gain, - 'digital_gain': digital_gain, - 'dpddata': dpddata, +internal_data = { + 'n_runs': 0, } results = { + 'adapt_dumps': [], + 'statplot': None, + 'modelplot': None, + 'modeldata': dpddata_to_str(dpddata), 'tx_median': 0, 'rx_median': 0, 'state': 'Idle', + 'stateprogress': 0, # in percent 'summary': ['DPD has not been calibrated yet'], } lock = Lock() command_queue = Queue(maxsize=1) +# Fill list of adapt dumps so that user can choose a previous +# setting across restarts. +results['adapt_dumps'].append("defaults") + +adapt_dump_files = glob.glob(os.path.join(plot_path, "adapt_*.pkl")) +re_adaptfile = re.compile(r"adapt_(.*)\.pkl") +for f in adapt_dump_files: + match = re_adaptfile.search(f) + if match: + results['adapt_dumps'].append(match.group(1)) + # Automatic Gain Control for the RX gain agc = Agc(meas, adapt, c) +def clear_pngs(results): + results['statplot'] = None + results['modelplot'] = None + pngs = glob.glob(os.path.join(plot_path, "*.png")) + for png in pngs: + try: + os.remove(png) + except: + results['summary'] += ["failed to delete " + png] + def engine_worker(): - try: - while True: + extStat = None + while True: + try: cmd = command_queue.get() if cmd == "quit": break elif cmd == "calibrate": with lock: - results['state'] = 'rx gain calibration' + results['state'] = 'RX Gain Calibration' + results['stateprogress'] = 0 + clear_pngs(results) - agc_success, agc_summary = agc.run() - summary = ["First calibration run:"] + agc_summary.split("\n") - if agc_success: + summary = [] + N_ITER = 3 + for i in range(N_ITER): agc_success, agc_summary = agc.run() - summary += ["Second calibration run: "] + agc_summary.split("\n") + summary += ["Iteration {}:".format(i)] + agc_summary.split("\n") + + with lock: + results['stateprogress'] = int((i + 1) * 100/N_ITER) + results['summary'] = ["Calibration ongoing:"] + summary + + if not agc_success: + break txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples() with lock: - settings['rx_gain'] = adapt.get_rxgain() - settings['digital_gain'] = adapt.get_digital_gain() results['tx_median'] = float(tx_median) results['rx_median'] = float(rx_median) results['state'] = 'Idle' - results['summary'] = ["Calibration was done:"] + summary + results['stateprogress'] = 100 + results['summary'] = summary + ["Calibration done"] + elif cmd == "reset": + model.reset_coefs() + with lock: + internal_data['n_runs'] = 0 + results['state'] = 'Idle' + results['stateprogress'] = 0 + results['summary'] = ["Reset"] + results['modeldata'] = dpddata_to_str(model.get_dpd_data()) + clear_pngs(results) + extStat = None + elif cmd == "trigger_run": + with lock: + results['state'] = 'Capture + Model' + results['stateprogress'] = 0 + n_runs = internal_data['n_runs'] + + while True: + # Get Samples and check gain + txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples() + + if extStat is None: + # At first run, we must decide how to create the bins + peak_estimated = tx_median * c.median_to_peak + extStat = ExtractStatistic(c, peak_estimated) + + with lock: + results['stateprogress'] += 2 + + # Extract usable data from measurement + tx, rx, phase_diff, n_per_bin = extStat.extract(txframe_aligned, rxframe_aligned) + + utctime = datetime.datetime.utcnow() + plot_file = "stats_{}.png".format(utctime.strftime("%s")) + extStat.plot(os.path.join(plot_path, plot_file), utctime.strftime("%Y-%m-%dT%H%M%S")) + n_meas = Heuristics.get_n_meas(n_runs) + + with lock: + results['statplot'] = "dpd/" + plot_file + results['stateprogress'] += 2 + results['summary'] = ["Captured {} samples".format(len(txframe_aligned)), + "TX/RX median: {} / {}".format(tx_median, rx_median), + extStat.get_bin_info(), + "Extracted Statistics: TX median={} RX median={}".format(tx_median, rx_median), + "Runs: {}/{}".format(extStat.n_meas, n_meas)] + if extStat.n_meas >= n_meas: + break + + if any(x is None for x in [tx, rx, phase_diff]): + with lock: + results['summary'] += ["Error! No data to calculate model"] + results['state'] = 'Idle' + results['stateprogress'] = 0 + else: + with lock: + results['state'] = 'Capture + Model' + results['stateprogress'] = 80 + results['summary'] += ["Training model"] + + model.train(tx, rx, phase_diff, lr=Heuristics.get_learning_rate(n_runs)) + + utctime = datetime.datetime.utcnow() + model_plot_file = "model_{}.png".format(utctime.strftime("%s")) + model.plot( + os.path.join(plot_path, model_plot_file), + utctime.strftime("%Y-%m-%dT%H%M%S")) + + with lock: + results['modelplot'] = "dpd/" + model_plot_file + results['state'] = 'Capture + Model' + results['stateprogress'] = 85 + results['summary'] += ["Getting DPD data"] + + dpddata = model.get_dpd_data() + with lock: + internal_data['dpddata'] = dpddata + internal_data['n_runs'] = 0 + + results['modeldata'] = dpddata_to_str(dpddata) + results['state'] = 'Capture + Model' + results['stateprogress'] = 90 + results['summary'] += ["Reset statistics"] + + extStat = None + + with lock: + results['state'] = 'Idle' + results['stateprogress'] = 100 + results['summary'] += ["New DPD coefficients calculated"] + elif cmd == "adapt": + with lock: + dpddata = internal_data['dpddata'] + results['state'] = 'Update Predistorter' + results['stateprogress'] = 50 + results['summary'] = [""] + iteration = internal_data['n_runs'] + internal_data['n_runs'] += 1 + + adapt.set_predistorter(dpddata) - finally: - with lock: - results['state'] = 'terminated' + time.sleep(2) + + txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples() + + # Store all settings for pre-distortion, tx and rx + utctime = datetime.datetime.utcnow() + dump_file = "adapt_{}.pkl".format(utctime.strftime("%s")) + adapt.dump(os.path.join(plot_path, dump_file)) + + with lock: + results['adapt_dumps'].append(utctime.strftime("%s")) + + # Collect logging data + off = symbol_align.calc_offset(txframe_aligned) + tx_mer = mer.calc_mer(txframe_aligned[off:off + c.T_U], debug_name='TX') + rx_mer = mer.calc_mer(rxframe_aligned[off:off + c.T_U], debug_name='RX') + mse = np.mean(np.abs((txframe_aligned - rxframe_aligned) ** 2)) + tx_gain = adapt.get_txgain() + rx_gain = adapt.get_rxgain() + digital_gain = adapt.get_digital_gain() + rx_shoulder_tuple = meas_shoulders.average_shoulders(rxframe_aligned) + tx_shoulder_tuple = meas_shoulders.average_shoulders(txframe_aligned) + + lr = Heuristics.get_learning_rate(iteration) + + summary = [f"Set predistorter:", + f"Signal measurements after iteration {iteration} with learning rate {lr}", + f"TX MER {tx_mer:.2}, RX MER {rx_mer:.2}", + f"Mean-square error: {mse:.3}"] + if tx_shoulder_tuple is not None: + summary.append("Shoulders: TX {!r}, RX {!r}".format(tx_shoulder_tuple, rx_shoulder_tuple)) + summary.append(f"Running with digital gain {digital_gain}, TX gain {tx_gain} and RX gain {rx_gain}") + + with lock: + results['state'] = 'Update Predistorter' + results['stateprogress'] = 100 + results['summary'] = ["Signal measurements after predistortion update"] + summary + elif cmd.startswith("restore_dump-"): + _, _, dump_id = cmd.partition("-") + if dump_id == "defaults": + model.reset_coefs() + dpddata = model.get_dpd_data() + adapt.set_predistorter(dpddata) + + tx_gain = adapt.get_txgain() + rx_gain = adapt.get_rxgain() + digital_gain = adapt.get_digital_gain() + with lock: + results['state'] = 'Idle' + results['stateprogress'] = 100 + results['summary'] = [f"Restored DPD defaults", + f"Running with digital gain {digital_gain}, TX gain {tx_gain} and RX gain {rx_gain}"] + results['modeldata'] = dpddata_to_str(dpddata) + else: + dump_file = os.path.join(plot_path, f"adapt_{dump_id}.pkl") + try: + d = adapt.restore(dump_file) + logging.info(f"Restore: {d}") + model.set_dpd_data(d['dpddata']) + with lock: + results['state'] = 'Idle' + results['stateprogress'] = 100 + results['summary'] = [f"Restored DPD settings from dumpfile {dump_id}", + f"Running with digital gain {d['digital_gain']}, TX gain {d['txgain']} and RX gain {d['rxgain']}"] + results['modeldata'] = dpddata_to_str(d["dpddata"]) + except: + e = traceback.format_exc() + with lock: + results['state'] = 'Idle' + results['stateprogress'] = 100 + results['summary'] = [f"Failed to restore DPD settings from dumpfile {dump_id}", + f"Error: {e}"] + except: + e = traceback.format_exc() + logging.error(e) + with lock: + results['summary'] = [f"Exception:"] + e.split("\n") + results['state'] = 'Autorestart pending' + results['stateprogress'] = 0 + + for i in range(5): + time.sleep(2) + with lock: + results['stateprogress'] += 20 + time.sleep(2) + with lock: + dt = datetime.datetime.utcnow().isoformat() + results['summary'] = [f"DPD engine auto-restarted at {dt} UTC", f"After exception {e}"] + results['state'] = 'Idle' + results['stateprogress'] = 0 engine = Thread(target=engine_worker) @@ -186,7 +410,7 @@ try: try: addr, msg_id, method, params = cmd_socket.receive_request() except ValueError as e: - logging.warning('YAML-RPC request error: {}'.format(e)) + logging.warning('RPC request error: {}'.format(e)) continue except TimeoutError: continue @@ -194,28 +418,24 @@ try: logging.info('Caught KeyboardInterrupt') break except: - logging.error('YAML-RPC unknown error') + logging.error('RPC unknown error') break - if method == 'trigger_run': - logging.info('YAML-RPC request : {}'.format(method)) - command_queue.put('trigger_run') - elif method == 'reset': - logging.info('YAML-RPC request : {}'.format(method)) - command_queue.put('reset') - elif method == 'set_setting': - logging.info('YAML-RPC request : {} -> {}'.format(method, params)) - # params == {'setting': ..., 'value': ...} - pass - elif method == 'get_settings': - with lock: - cmd_socket.send_success_response(addr, msg_id, settings) + if any(method == m for m in ['trigger_run', 'reset', 'adapt']): + logging.info('Received RPC request : {}'.format(method)) + command_queue.put(method) + cmd_socket.send_success_response(addr, msg_id, None) + elif method == 'restore_dump': + logging.info('Received RPC request : restore_dump({})'.format(params['dump_id'])) + command_queue.put(f"restore_dump-{params['dump_id']}") + cmd_socket.send_success_response(addr, msg_id, None) elif method == 'get_results': with lock: cmd_socket.send_success_response(addr, msg_id, results) elif method == 'calibrate': - logging.info('YAML-RPC request : {}'.format(method)) + logging.info('Received RPC request : {}'.format(method)) command_queue.put('calibrate') + cmd_socket.send_success_response(addr, msg_id, None) else: cmd_socket.send_error_response(addr, msg_id, "request not understood") finally: @@ -346,7 +566,7 @@ while i < num_iter: # The MIT License (MIT) # # Copyright (c) 2017 Andreas Steger -# Copyright (c) 2018 Matthias P. Braendli +# Copyright (c) 2019 Matthias P. Braendli # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/python/gui-dpdce.ini b/python/gui-dpdce.ini index 48c6abf..4385c80 100644 --- a/python/gui-dpdce.ini +++ b/python/gui-dpdce.ini @@ -19,7 +19,10 @@ samps=81920 coef_file=poly.coef # Write logs to this folder, or leave empty for no logs -log_folder= +logs_directory= + +# Saving plots to the static directory makes them accessible to the browser +plot_directory=gui/static/dpd [gui] diff --git a/python/gui.py b/python/gui.py index ce7948c..a9328ee 100755 --- a/python/gui.py +++ b/python/gui.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (C) 2018 @@ -33,6 +33,7 @@ from lib import zmqrc env = Environment(loader=FileSystemLoader('gui/templates')) base_js = ["js/odr.js"] +base_css = ["css/odr.css"] class Root: def __init__(self, dpd_port): @@ -51,7 +52,8 @@ class Root: @cherrypy.expose def home(self): tmpl = env.get_template("home.html") - return tmpl.render(tab='home', js=base_js, is_login=False) + js = base_js + ["js/odr-home.js"] + return tmpl.render(tab='home', js=js, css=base_css, is_login=False) @cherrypy.expose def rcvalues(self): diff --git a/python/gui/api.py b/python/gui/api.py index bff224e..f9e0ad0 100755 --- a/python/gui/api.py +++ b/python/gui/api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2018 +# Copyright (C) 2019 # Matthias P. Braendli, matthias.braendli@mpb.li # # http://www.opendigitalradio.org @@ -80,8 +80,18 @@ class API: return send_error(str(e)) return send_ok() else: - cherrypy.response.status = 400 - return send_error("POST only") + if all(p in kwargs for p in ('controllable', 'param')): + try: + return send_ok(self.mod_rc.get_param_value(kwargs['controllable'], kwargs['param'])) + except IOError as e: + cherrypy.response.status = 503 + return send_error(str(e)) + except ValueError as e: + cherrypy.response.status = 503 + return send_error(str(e)) + else: + cherrypy.response.status = 400 + return send_error("missing 'controllable' or 'param' GET parameters") def _wrap_dpd(self, method, data=None): try: @@ -89,12 +99,12 @@ class API: return send_ok(reply) except ValueError as e: cherrypy.response.status = 503 - return send_error("YAML-RPC call error: {}".format(e)) + return send_error("DPDCE remote procedure call error: {}".format(e)) except TimeoutError as e: cherrypy.response.status = 503 - return send_error("YAML-RPC timeout: {}".format(e)) + return send_error("DPDCE remote procedure call timed out") cherrypy.response.status = 500 - return send_error("YAML-RPC unknown error") + return send_error("Unknown DPDCE remote procedure error error") @cherrypy.expose @cherrypy.tools.json_out() @@ -107,6 +117,15 @@ class API: @cherrypy.expose @cherrypy.tools.json_out() + def dpd_adapt(self, **kwargs): + if cherrypy.request.method == 'POST': + return self._wrap_dpd("adapt") + else: + cherrypy.response.status = 400 + return send_error("POST only") + + @cherrypy.expose + @cherrypy.tools.json_out() def dpd_reset(self, **kwargs): if cherrypy.request.method == 'POST': return self._wrap_dpd("reset") @@ -116,12 +135,20 @@ class API: @cherrypy.expose @cherrypy.tools.json_out() - def dpd_settings(self, setting: str, value: str, **kwargs): + def dpd_restore_dump(self, **kwargs): if cherrypy.request.method == 'POST': - data = {'setting': setting, 'value': value} - return self._wrap_dpd("set_setting", data) + cl = cherrypy.request.headers['Content-Length'] + rawbody = cherrypy.request.body.read(int(cl)) + params = json.loads(rawbody.decode()) + if 'dump_id' in params: + data = {'dump_id': params['dump_id']} + return self._wrap_dpd("restore_dump", data) + else: + cherrypy.response.status = 400 + return send_error("Missing dump_id") else: - return self._wrap_dpd("get_settings") + cherrypy.response.status = 400 + return send_error("POST only") @cherrypy.expose @cherrypy.tools.json_out() diff --git a/python/gui/static/css/odr.css b/python/gui/static/css/odr.css new file mode 100644 index 0000000..1710464 --- /dev/null +++ b/python/gui/static/css/odr.css @@ -0,0 +1,14 @@ +.glyphicon-refresh-animate { + -animation: spin 1.8s infinite linear; + -webkit-animation: spin2 1.8s infinite linear; +} + +@-webkit-keyframes spin2 { + from { -webkit-transform: rotate(0deg);} + to { -webkit-transform: rotate(360deg);} +} + +@keyframes spin { + from { transform: scale(1) rotate(0deg);} + to { transform: scale(1) rotate(360deg);} +} diff --git a/python/gui/static/js/odr-home.js b/python/gui/static/js/odr-home.js new file mode 100644 index 0000000..b74c4c8 --- /dev/null +++ b/python/gui/static/js/odr-home.js @@ -0,0 +1,240 @@ +// Copyright (C) 2019 +// Matthias P. Braendli, matthias.braendli@mpb.li +// +// http://www.opendigitalradio.org +// +// This file is part of ODR-DabMod. +// +// ODR-DabMod is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// ODR-DabMod is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +// + +function apiRequestChain(uri, get_data, success_callback, fail_callback) { + $.ajax({ + type: "GET", + url: uri, + data: get_data, + contentType: 'application/json', + dataType: 'json', + + error: function(data) { + console.log("GET " + JSON.stringify(get_data) + " error: " + data.responseText); + fail_callback(data.responseText); + }, + success: function(data) { + if (data.status == 'ok') { + success_callback(data.data); + } + else { + fail_callback(data.data); + } + }, + }); +} + +function mark_pending(id, comment) { + document.getElementById(id).className = "glyphicon glyphicon-refresh glyphicon-refresh-animate"; + + if (comment) { + document.getElementById(id + "_comment").innerHTML = comment; + } +} + +var failure_encountered = false; + +function mark_overall_ok() { + if (!failure_encountered) { + document.getElementById("overall_state").className = "glyphicon glyphicon-ok"; + } +} + +function mark_ok(id, comment) { + document.getElementById(id).className = "glyphicon glyphicon-ok"; + + if (comment) { + document.getElementById(id + "_comment").innerHTML = comment; + } +} + +function mark_fail(id, reason) { + failure_encountered = true; + + var el = document.getElementById(id); + el.className = "glyphicon glyphicon-remove"; + el.style.color = "#FF3333"; + + document.getElementById(id + "_comment").innerHTML = reason; + + var overall = document.getElementById("overall_state"); + overall.style.color = "#FF8833"; + overall.className = "glyphicon glyphicon-alert"; +} + +function check_rc() { + mark_pending('is_rc_ok'); + apiRequestChain("/api/parameter", + {controllable: 'sdr', param: 'freq'}, + function(data) { + mark_ok('is_rc_ok'); + check_modulating(0); + }, + function(data) { + mark_fail('is_rc_ok', JSON.parse(data)['reason']); + }); +} + +function check_modulating(last_num_frames) { + mark_pending('is_modulating'); + apiRequestChain("/api/parameter", + {controllable: 'sdr', param: 'frames'}, + function(data) { + if (data > 0) { + if (last_num_frames == 0) { + setTimeout(function() { check_modulating(data); }, 200); + } + else { + if (data == last_num_frames) { + mark_fail('is_modulating', "Frame counter not incrementing: " + data); + } + else { + mark_ok('is_modulating', "Number of frames modulated: " + data); + } + check_gpsdo_ok(); + } + } + else { + mark_fail('is_modulating', 'number of frames is 0'); + } + }, + function(data) { + mark_fail('is_modulating', data); + }); +} + +function check_gpsdo_ok() { + mark_pending('is_gpsdo_ok'); + apiRequestChain("/api/parameter", + {controllable: 'sdr', param: 'gpsdo_num_sv'}, + function(data) { + if (data > 3) { + mark_ok('is_gpsdo_ok', "Number of SVs used: " + data); + } + else { + mark_fail('is_gpsdo_ok', "Number of SVs (" + data + ") is too low"); + } + check_underrunning(0, 0); + check_late(0, 0); + }, + function(data) { + mark_fail('is_gpsdo_ok', json.parse(data)['reason']); + }); +} + + +function check_underrunning(iteration, first_underruns) { + var n_checks = 3; + + apiRequestChain("/api/parameter", + {controllable: 'sdr', param: 'underruns'}, + function(data) { + if (iteration == 0) { + mark_pending('is_underrunning', "Checking for underruns"); + setTimeout(function() { check_underrunning(iteration+1, data); }, 2000); + } + else if (iteration < n_checks) { + mark_pending('is_underrunning', "Check " + iteration + "/" + n_checks + "..."); + setTimeout(function() { check_underrunning(iteration+1, first_underruns); }, 2000); + } + else { + if (data == first_underruns) { + mark_ok('is_underrunning', "Number of underruns is not increasing: " + data); + } + else { + mark_fail('is_underrunning', "Underruns observed in last " + n_checks + " seconds: " + data); + } + check_rate_4x(); + } + }, + function(data) { + mark_fail('is_underrunning', data); + }); +} + +function check_late(iteration, first_late) { + var n_checks = 3; + + apiRequestChain("/api/parameter", + {controllable: 'sdr', param: 'latepackets'}, + function(data) { + if (iteration == 0) { + mark_pending('is_late', "Checking for late packets"); + setTimeout(function() { check_late(iteration+1, data); }, 2000); + } + else if (iteration < n_checks) { + mark_pending('is_late', "Check " + iteration + "/" + n_checks + "..."); + setTimeout(function() { check_late(iteration+1, first_late); }, 2000); + } + else { + if (data == first_late) { + mark_ok('is_late', "Number of late packets is not increasing: " + data); + } + else { + mark_fail('is_late', "Late packets observed in last " + n_checks + " seconds: " + data); + } + } + }, + function(data) { + mark_fail('is_late', data); + }); +} + +function check_rate_4x() { + mark_pending('is_rate_4x'); + apiRequestChain("/api/parameter", + {controllable: 'modulator', param: 'rate'}, + function(data) { + if (data == 8192000) { + mark_ok('is_rate_4x', "Samplerate: " + data); + } + else { + mark_fail('is_rate_4x', "Samplerate is not 8192ksps: " + data); + } + check_dpdce_running(); + }, + function(data) { + mark_fail('is_rate_4x', JSON.parse(data)['reason']); + }); +} + +function check_dpdce_running() { + mark_pending('is_dpdce_running'); + apiRequestChain("/api/dpd_results", + {}, + function(data) { + mark_ok('is_dpdce_running', "State: " + data['state']); + mark_overall_ok(); + }, + function(data) { + mark_fail('is_dpdce_running', JSON.parse(data)['reason']); + }); +} + +$(function(){ + setTimeout(check_rc, 20); +}); + + +// ToolTip init +$(function(){ + $('[data-toggle="tooltip"]').tooltip(); +}); diff --git a/python/gui/static/js/odr-predistortion.js b/python/gui/static/js/odr-predistortion.js index 04d2773..4dae068 100644 --- a/python/gui/static/js/odr-predistortion.js +++ b/python/gui/static/js/odr-predistortion.js @@ -1,4 +1,4 @@ -// Copyright (C) 2018 +// Copyright (C) 2019 // Matthias P. Braendli, matthias.braendli@mpb.li // // http://www.opendigitalradio.org @@ -18,6 +18,8 @@ // You should have received a copy of the GNU General Public License // along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +var adapt_dumps = []; + function resultrefresh() { var jqxhr = doApiRequestGET("/api/dpd_results", function(data) { var summary = ""; @@ -29,6 +31,30 @@ function resultrefresh() { $('#dpdresults').html(summary); $('#dpdstatus').text(data['state']); + var percentage = data['stateprogress']; + if (percentage > 100) { + percentage = 100; + } + $('#dpdprogress').css('width', percentage + '%'); + $('#dpdprogresstext').text(percentage + '%'); + + if (data['statplot']) { + $('#dpdcapturestats').attr('src', data['statplot']); + } + else { + $('#dpdcapturestats').attr('src', ""); + } + + $('#dpdmodeldata').html(data['modeldata']); + + if (data['modelplot']) { + $('#dpdmodelplot').attr('src', data['modelplot']); + } + else { + $('#dpdmodelplot').attr('src', ""); + } + + adapt_dumps = data['adapt_dumps']; }); jqxhr.always(function() { @@ -36,8 +62,32 @@ function resultrefresh() { }); } +function adaptdumpsrefresh() { + $('#dpdadaptdumps').html(""); + + $.each(adapt_dumps, function(i, item) { + console.log(item); + + if (isNaN(+item)) { + $('#dpdadaptdumps').append($('<option>', { + value: item, + text : "DPD settings from " + item, + })); + } + else { + var d = new Date(0); + d.setUTCSeconds(item); + + $('#dpdadaptdumps').append($('<option>', { + value: item, + text : "DPD settings from " + d.toISOString(), + })); + } + }); +} + $(function(){ - setTimeout(resultrefresh, 2000); + setTimeout(resultrefresh, 20); $('#calibratebtn').click(function() { doApiRequestPOST("/api/dpd_calibrate", {}, function(data) { @@ -45,72 +95,39 @@ $(function(){ }); }); -}); - -/* -function calibraterefresh() { - doApiRequestGET("/api/calibrate", function(data) { - var text = "Captured TX signal and feedback." + - " TX median: " + data['tx_median'] + - " RX median: " + data['rx_median'] + - " with relative timestamp offset " + - (data['tx_ts'] - data['rx_ts']) + - " and measured offset " + data['coarse_offset'] + - ". Correlation: " + data['correlation']; - $('#calibrationresults').text(text); + $('#triggerbtn').click(function() { + doApiRequestPOST("/api/dpd_trigger_run", {}, function(data) { + console.log("run succeeded: " + JSON.stringify(data)); + }); }); -} -$(function(){ - $('#refreshframesbtn').click(function() { - var d = new Date(); - var n = d.getTime(); - $('#txframeimg').src = "dpd/txframe.png?cachebreak=" + n; - $('#rxframeimg').src = "dpd/rxframe.png?cachebreak=" + n; + $('#adaptbtn').click(function() { + doApiRequestPOST("/api/dpd_adapt", {}, function(data) { + console.log("adapt succeeded: " + JSON.stringify(data)); + }); }); - $('#capturebutton').click(function() { - doApiRequestPOST("/api/trigger_capture", {}, function(data) { - console.log("trigger_capture succeeded: " + JSON.stringify(data)); + + $('#resetbtn').click(function() { + doApiRequestPOST("/api/dpd_reset", {}, function(data) { + console.log("reset succeeded: " + JSON.stringify(data)); }); }); - $('#dpdstatusbutton').click(function() { - doApiRequestGET("/api/dpd_status", function(data) { - console.log("dpd_status succeeded: " + JSON.stringify(data)); - $('#histogram').text(data.histogram); - $('#capturestatus').text(data.capture.status); - $('#capturelength').text(data.capture.length); - $('#tx_median').text(data.capture.tx_median); - $('#rx_median').text(data.capture.rx_median); - }); + $('#adaptdumpsrefreshbtn').click(adaptdumpsrefresh); + + $('#adaptdumpsload').click(function() { + var elt = document.getElementById("dpdadaptdumps"); - $.ajax({ - type: "GET", - url: "/api/dpd_capture_pointcloud", - - error: function(data) { - if (data.status == 500) { - var errorWindow = window.open("", "_self"); - errorWindow.document.write(data.responseText); - } - else { - $.gritter.add({ title: 'API', - text: "AJAX failed: " + data.statusText, - image: '/fonts/warning.png', - sticky: true, - }); - } - }, - success: function(data) { - $('#dpd_pointcloud').value(data) + if (elt.selectedIndex != -1) { + var selectedoption = elt.options[elt.selectedIndex].value; + doApiRequestPOST("/api/dpd_restore_dump", {dump_id: selectedoption}, function(data) { + console.log("reset succeeded: " + JSON.stringify(data)); + }); } - }) }); }); -*/ - // ToolTip init $(function(){ diff --git a/python/gui/static/js/odr-rcvalues.js b/python/gui/static/js/odr-rcvalues.js index f49674c..486a8a7 100644 --- a/python/gui/static/js/odr-rcvalues.js +++ b/python/gui/static/js/odr-rcvalues.js @@ -31,22 +31,28 @@ function requestStatus() { doApiRequestGET("/api/rc_parameters", function(data) { console.log(data); - let keys = Object.keys(data); - keys.sort(); + let controllable_names = Object.keys(data); + controllable_names.sort(); var key1; - for (key1 in keys) { - let keys2 = Object.keys(data[keys[key1]]); - keys2.sort(); + for (key1 in controllable_names) { + let param_names = Object.keys(data[controllable_names[key1]]); + param_names.sort(); var key2; - for (key2 in keys2) { - var param = data[keys[key1]][keys2[key2]]; - var key = keys[key1] + "_" + keys2[key2]; + for (key2 in param_names) { + var name_controllable = controllable_names[key1]; + var name_param = param_names[key2]; + var key = name_controllable + "_" + name_param; + + var param = data[name_controllable][name_param]; var valueentry = '<input type="text" id="input'+key+'" ' + 'value="' + param['value'] + '">' + '<button type="button" class="btn btn-xs btn-warning"' + - 'id="button'+key+'" >upd</button>'; + 'id="button'+key+'" ' + + 'data-controllable="'+name_controllable+'" ' + + 'data-param="'+name_param+'" ' + + '>upd</button>'; $('#rctable > tbody:last').append( '<tr><td>'+key+'</td>'+ @@ -54,7 +60,10 @@ function requestStatus() { '<td>'+param['help']+'</td></tr>'); $('#button'+key).click(function() { - buttonSetRc("input"+key, key1, key2); + var attr_c = this.getAttribute('data-controllable'); + var attr_p = this.getAttribute('data-param'); + var k = attr_c + "_" + attr_p; + buttonSetRc("input"+k, attr_c, attr_p); }); } } diff --git a/python/gui/templates/about.html b/python/gui/templates/about.html index 3a05230..b781d54 100644 --- a/python/gui/templates/about.html +++ b/python/gui/templates/about.html @@ -1,24 +1,3 @@ -<!-- - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. ---> - - <!DOCTYPE html> <html lang="en"> @@ -63,3 +42,24 @@ along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. </html> +<!-- + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +--> + + diff --git a/python/gui/templates/home.html b/python/gui/templates/home.html index 5cb29f8..398df37 100644 --- a/python/gui/templates/home.html +++ b/python/gui/templates/home.html @@ -1,3 +1,59 @@ +<!DOCTYPE html> +<html lang="en"> + +{% include 'head.html' %} + +<body> + {% include 'body-nav.html' %} + + <div class="container-fluid"> + <div class="jumbotron"> + <h1>Opendigitalradio</h1><h2>ODR-DabMod Status Check + <span id="overall_state" class="glyphicon glyphicon-refresh glyphicon-refresh-animate"></span> + </h2> + <div class="well well-sm"> + <p>ODR-DabMod + </p> + <ul> + <li>Answering to RC: + <span id="is_rc_ok" class="glyphicon glyphicon-question-sign"></span> + <span id="is_rc_ok_comment"><span> + </li> + <li>Frame generation: + <span id="is_modulating" class="glyphicon glyphicon-question-sign"></span> + <span id="is_modulating_comment"><span> + </li> + <li>GPSDO status: + <span id="is_gpsdo_ok" class="glyphicon glyphicon-question-sign"></span> + <span id="is_gpsdo_ok_comment"><span> + </li> + <li>Underruns: + <span id="is_underrunning" class="glyphicon glyphicon-question-sign"></span> + <span id="is_underrunning_comment"><span> + </li> + <li>Late packets: + <span id="is_late" class="glyphicon glyphicon-question-sign"></span> + <span id="is_late_comment"><span> + </li> + </ul> + + <p>Checking predistortion + <ul> + <li>Sample rate at 4x native rate: + <span id="is_rate_4x" class="glyphicon glyphicon-question-sign"></span> + <span id="is_rate_4x_comment"><span> + </li> + <li>DPDCE running: + <span id="is_dpdce_running" class="glyphicon glyphicon-question-sign"></span> + <span id="is_dpdce_running_comment"><span> + </li> + </ul> + </div> + </div> + </div> +</body> +</html> + <!-- Copyright (C) 2018 Matthias P. Braendli, matthias.braendli@mpb.li @@ -17,19 +73,3 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. --> - -<!DOCTYPE html> -<html lang="en"> - -{% include 'head.html' %} - -<body> - {% include 'body-nav.html' %} - - <div class="container-fluid"> - <div class="jumbotron"> - <h1>Opendigitalradio</h1><h2>ODR-DabMod Interface</h2> - </div> - </div> -</body> -</html> diff --git a/python/gui/templates/modulator.html b/python/gui/templates/modulator.html index 6deffb1..016344a 100644 --- a/python/gui/templates/modulator.html +++ b/python/gui/templates/modulator.html @@ -1,23 +1,3 @@ -<!-- - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. ---> - <!DOCTYPE html> <html lang="en"> @@ -71,3 +51,23 @@ along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. </div> </body> </html> + +<!-- + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/gui/templates/predistortion.html b/python/gui/templates/predistortion.html index cc5ecb0..62e8503 100644 --- a/python/gui/templates/predistortion.html +++ b/python/gui/templates/predistortion.html @@ -1,23 +1,3 @@ -<!-- - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. ---> - <!DOCTYPE html> <html lang="en"> @@ -31,42 +11,87 @@ along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. <div class="panel panel-default"> <div class="panel-heading">Status and calibration</div> <div class="panel-body"> - <div>Current DPDCE status: <span id="dpdstatus" style="font-weight:bold;">N/A</span> - <div class="well well-sm" id="dpdresults">N/A</div> + + <div class="container-fluid"> + <div class="row"> + <div class="col-sm-4"> + <h2>Current DPDCE status</h2> + <div> + <div id="dpdstatus" style="font-weight:bold;">N/A</div> + <div class="progress"> + <div id="dpdprogress" class="progress-bar" role="progressbar" style="width:0%"> + <span id="dpdprogresstext"></span> + </div> + </div> + </div> + <div class="well well-sm" id="dpdresults">N/A</div> + </div> + <div class="col-sm-4"> + <h2>List of saved DPD settings</h2> + <!--TODO: 'erase' and 'clear' buttons. Show DPD settings in tooltip?--> + <p>This list contains previously used predistortion settings that you + can recall.</p> + <p> + <select id="dpdadaptdumps" size="8" style="width:70%" multiple></select> + </p> + <p> + <button type="button" class="btn btn-sm btn-info" id="adaptdumpsrefreshbtn">Refresh + </button> + <button type="button" class="btn btn-sm btn-warning" id="adaptdumpsload">Load and Apply + </button> + </p> + </div> + <div class="col-sm-4"> + <h2>Summary</h2> + <p>Calibration needs to be done once before the PA model + can be trained. Every time calibration is changed, the predistortion + parameters are invalidated!</p> + <p>Once calibration succeeded and correct RX gain is set, you + can trigger a capture and model the PA. Usually, several capture + runs are needed before the model can be trained.</p> + <p>The capture and model analysis will calculate a new set of + DPD model data, that you can apply using the Update Predistorter button.</p> + <p>The reset button allows you to reset the computation engine. It does not + modify the currently active predistorter.</p> + </div> + </div> </div> - <div>Calibration needs to be done once before the PA model - can be trained. Every time calibration is changed, the predistortion - parameters are invalidated!</div> <button type="button" class="btn btn-sm btn-warning" id="calibratebtn"> Calibrate</button> + <button type="button" class="btn btn-sm btn-warning" id="triggerbtn"> + Trigger Capture and PA Modeling</button> + <button type="button" class="btn btn-sm btn-warning" id="adaptbtn"> + Update Predistorter</button> + <button type="button" class="btn btn-sm btn-info" id="resetbtn"> + Reset Capture and Model</button> </div> </div> - <!-- + <div class="panel panel-default"> - <div class="panel-heading">Capture TX and RX frames</div> + <div class="panel-heading">Capture Statistics</div> <div class="panel-body"> - <div> - <img id="txframeimg" src="dpd/txframe.png" width="320" height="240" /> - <img id="rxframeimg" src="dpd/rxframe.png" width="320" height="240" /> - </div> - <div> - <button type="button" class="btn btn-sm btn-info" id="refreshframesbtn"> - Refresh</button> - </div> + <img id="dpdcapturestats" /> </div> </div> - <div class="panel panel-default"> - <div class="panel-heading">Capture</div> + <div class="panel-heading">AM/AM and AM/PM Model</div> <div class="panel-body"> - <div>On pressing this button, - the DPDCE will trigger a capture and a quick data - analysis, without updating any DPD models.</div> - <button type="button" class="btn btn-sm btn-info" id="capturebutton"> - Capture</button> + <div class="container-fluid"> + <div class="row"> + <div class="col-sm-2"> + <p>Model data:</p> + </div> + <div class="col-sm-10"> + <pre id="dpdmodeldata"></pre> + </div> + </div> + </div> + <img id="dpdmodelplot" /> </div> </div> + + <!-- <div class="panel panel-default"> <div class="panel-heading">Status</div> <div class="panel-body"> @@ -87,3 +112,23 @@ along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. </div> </body> </html> + +<!-- + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/gui/templates/rcvalues.html b/python/gui/templates/rcvalues.html index c1786bc..9e607bc 100644 --- a/python/gui/templates/rcvalues.html +++ b/python/gui/templates/rcvalues.html @@ -1,23 +1,3 @@ -<!-- - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - -This file is part of ODR-DabMod. - -ODR-DabMod is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -ODR-DabMod is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. ---> - <!DOCTYPE html> <html lang="en"> @@ -46,3 +26,22 @@ along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. </div> </body> </html> +<!-- + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + +This file is part of ODR-DabMod. + +ODR-DabMod is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +ODR-DabMod is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. +--> diff --git a/python/lib/yamlrpc.py b/python/lib/yamlrpc.py index d963601..67c65ff 100644 --- a/python/lib/yamlrpc.py +++ b/python/lib/yamlrpc.py @@ -23,6 +23,9 @@ """yamlrpc is json-rpc, except that it's yaml and not json.""" +# This maybe won't work over ethernet, but for localhost it's ok +UDP_PACKETSIZE = 2048 + # Same as jsonrpc version we're aiming to mirror in YAML YAMLRPC_VERSION = "2.0" @@ -80,9 +83,9 @@ class Socket: def receive_response(self, expected_msg_id: int): try: - data, addr = self.socket.recvfrom(512) + data, addr = self.socket.recvfrom(UDP_PACKETSIZE) except socket.timeout as to: - raise TimeoutError("Timeout: " + str(to)) + raise TimeoutError() y = yaml.load(data.decode()) @@ -117,7 +120,7 @@ class Socket: def receive_request(self): try: - data, addr = self.socket.recvfrom(512) + data, addr = self.socket.recvfrom(UDP_PACKETSIZE) except socket.timeout as to: raise TimeoutError("Timeout: " + str(to)) diff --git a/python/lib/zmqrc.py b/python/lib/zmqrc.py index 3897d7a..423f91d 100644 --- a/python/lib/zmqrc.py +++ b/python/lib/zmqrc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (C) 2018 @@ -22,15 +22,16 @@ # along with ODR-DabMod. If not, see <http://www.gnu.org/licenses/>. import zmq import json +from typing import List -class ModRemoteControl(object): +class ModRemoteControl: """Interact with ODR-DabMod using the ZMQ RC""" def __init__(self, mod_host, mod_port=9400): self._host = mod_host self._port = mod_port self._ctx = zmq.Context() - def _read(self, message_parts): + def _read(self, message_parts: List[str]): sock = zmq.Socket(self._ctx, zmq.REQ) sock.setsockopt(zmq.LINGER, 0) sock.connect("tcp://{}:{}".format(self._host, self._port)) @@ -70,14 +71,14 @@ class ModRemoteControl(object): return modules - def get_param_value(self, module, param): + def get_param_value(self, module: str, param: str) -> str: value = self._read(['get', module, param]) if value[0] == 'fail': raise ValueError("Error getting param: {}".format(value[1])) else: return value[0] - def set_param_value(self, module, param, value): + def set_param_value(self, module: str, param: str, value: str) -> None: ret = self._read(['set', module, param, value]) if ret[0] == 'fail': raise ValueError("Error setting param: {}".format(ret[1])) |