GUI: Add ability to restore previous DPD settings

6 files changed, 169 insertions, 104 deletions

diff --git a/python/dpd/Adapt.py b/python/dpd/Adapt.py
index 9c2c2b6..8023a53 100644
--- a/python/dpd/Adapt.py
+++ b/python/dpd/Adapt.py
@@ -152,7 +152,7 @@ class Adapt:
             _write_lut_file(scalefactor, lut, self._coef_path)
         else:
             raise ValueError("Unknown predistorter '{}'".format(dpddata[0]))
-        self._mod_rc.set_param_value("memlesspoly", "coefffile", self._coef_path)
+        self._mod_rc.set_param_value("memlesspoly", "coeffile", self._coef_path)
 
     def dump(self, path: str) -> None:
         """Backup current settings to a file"""
@@ -161,26 +161,31 @@ class Adapt:
             "txgain": self.get_txgain(),
             "rxgain": self.get_rxgain(),
             "digital_gain": self.get_digital_gain(),
-            "predistorter": self.get_predistorter()
+            "dpddata": self.get_predistorter()
         }
 
         with open(path, "wb") as f:
             pickle.dump(d, f)
 
-    def load(self, path: str) -> None:
+    def restore(self, path: str):
         """Restore settings from a file"""
         with open(path, "rb") as f:
             d = pickle.load(f)
 
             self.set_txgain(0)
+
+            # If any of the following fail, we will be running
+            # with the safe value of txgain=0
             self.set_digital_gain(d["digital_gain"])
             self.set_rxgain(d["rxgain"])
-            self.set_predistorter(d["predistorter"])
+            self.set_predistorter(d["dpddata"])
             self.set_txgain(d["txgain"])
 
+            return d
+
 # The MIT License (MIT)
 #
-# Copyright (c) 2018 Matthias P. Braendli
+# Copyright (c) 2019 Matthias P. Braendli
 # Copyright (c) 2017 Andreas Steger
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
diff --git a/python/dpd/Model_Poly.py b/python/dpd/Model_Poly.py
index 7ab6aa1..ef3fed3 100644
--- a/python/dpd/Model_Poly.py
+++ b/python/dpd/Model_Poly.py
@@ -125,6 +125,11 @@ class Poly:
     def get_dpd_data(self):
         return "poly", self.coefs_am, self.coefs_pm
 
+    def set_dpd_data(self, dpddata):
+        if dpddata[0] != "poly" or len(dpddata) != 3:
+            raise ValueError("dpddata is not of 'poly' format")
+        _, self.coefs_am, self.coefs_pm = dpddata
+
     def _am_calc_line(self, coefs, min_amp, max_amp):
         rx_range = np.linspace(min_amp, max_amp)
         tx_est = np.sum(self._am_poly(rx_range) * coefs, axis=1)
diff --git a/python/dpdce.py b/python/dpdce.py
index 6c373fd..27c5253 100755
--- a/python/dpdce.py
+++ b/python/dpdce.py
@@ -87,6 +87,7 @@ import numpy as np
 import traceback
 import os.path
 import glob
+import re
 from threading import Thread, Lock
 from queue import Queue
 from dpd.Model import Poly
@@ -134,19 +135,14 @@ if cli_args.reset:
 cmd_socket = yamlrpc.Socket(bind_port=control_port)
 
 # The following is accessed by both threads and need to be locked
-settings = {
-        'rx_gain': rx_gain,
-        'tx_gain': tx_gain,
-        'digital_gain': digital_gain,
-        'dpddata': dpddata,
-        }
 internal_data = {
         'n_runs': 0,
         }
 results = {
+        'adapt_dumps': [],
         'statplot': None,
         'modelplot': None,
-        'modeldata': "",
+        'modeldata': repr(dpddata),
         'tx_median': 0,
         'rx_median': 0,
         'state': 'Idle',
@@ -156,6 +152,17 @@ results = {
 lock = Lock()
 command_queue = Queue(maxsize=1)
 
+# Fill list of adapt dumps so that user can choose a previous
+# setting across restarts.
+results['adapt_dumps'].append("defaults")
+
+adapt_dump_files = glob.glob(os.path.join(plot_path, "adapt_*.pkl"))
+re_adaptfile = re.compile(r"adapt_(.*)\.pkl")
+for f in adapt_dump_files:
+    match = re_adaptfile.search(f)
+    if match:
+        results['adapt_dumps'].append(match.group(1))
+
 # Automatic Gain Control for the RX gain
 agc = Agc(meas, adapt, c)
 
@@ -171,8 +178,8 @@ def clear_pngs(results):
 
 def engine_worker():
     extStat = None
-    try:
-        while True:
+    while True:
+        try:
             cmd = command_queue.get()
 
             if cmd == "quit":
@@ -184,7 +191,7 @@ def engine_worker():
                     clear_pngs(results)
 
                 summary = []
-                N_ITER = 5
+                N_ITER = 3
                 for i in range(N_ITER):
                     agc_success, agc_summary = agc.run()
                     summary += ["Iteration {}:".format(i)] + agc_summary.split("\n")
@@ -199,8 +206,6 @@ def engine_worker():
                 txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples()
 
                 with lock:
-                    settings['rx_gain'] = adapt.get_rxgain()
-                    settings['digital_gain'] = adapt.get_digital_gain()
                     results['tx_median'] = float(tx_median)
                     results['rx_median'] = float(rx_median)
                     results['state'] = 'Idle'
@@ -233,9 +238,6 @@ def engine_worker():
 
                     with lock:
                         results['stateprogress'] += 5
-                        results['summary'] = ["Captured {} samples".format(len(txframe_aligned)),
-                            "TX/RX median: {} / {}".format(tx_median, rx_median),
-                            extStat.get_bin_info()]
 
                     # Extract usable data from measurement
                     tx, rx, phase_diff, n_per_bin = extStat.extract(txframe_aligned, rxframe_aligned)
@@ -248,8 +250,11 @@ def engine_worker():
                     with lock:
                         results['statplot'] = "dpd/" + plot_file
                         results['stateprogress'] += 5
-                        results['summary'] += ["Extracted Statistics: TX median={} RX median={}".format(tx_median, rx_median),
-                                "Runs: {}/{}".format(extStat.n_meas, n_meas)]
+                        results['summary'] = ["Captured {} samples".format(len(txframe_aligned)),
+                            "TX/RX median: {} / {}".format(tx_median, rx_median),
+                            extStat.get_bin_info(),
+                            "Extracted Statistics: TX median={} RX median={}".format(tx_median, rx_median),
+                            "Runs: {}/{}".format(extStat.n_meas, n_meas)]
                     if extStat.n_meas >= n_meas:
                         break
 
@@ -303,7 +308,7 @@ def engine_worker():
                     iteration = internal_data['n_runs']
                     internal_data['n_runs'] += 1
 
-                answer = adapt.set_predistorter(dpddata)
+                adapt.set_predistorter(dpddata)
 
                 time.sleep(2)
 
@@ -314,6 +319,9 @@ def engine_worker():
                 dump_file = "adapt_{}.pkl".format(utctime.strftime("%s"))
                 adapt.dump(os.path.join(plot_path, dump_file))
 
+                with lock:
+                    results['adapt_dumps'].append(utctime.strftime("%s"))
+
                 # Collect logging data
                 off = symbol_align.calc_offset(txframe_aligned)
                 tx_mer = mer.calc_mer(txframe_aligned[off:off + c.T_U], debug_name='TX')
@@ -327,7 +335,7 @@ def engine_worker():
 
                 lr = Heuristics.get_learning_rate(iteration)
 
-                summary = [f"Set predistorter: {answer}",
+                summary = [f"Set predistorter:",
                         f"Signal measurements after iteration {iteration} with learning rate {lr}",
                         f"TX MER {tx_mer}, RX MER {rx_mer}",
                         "Shoulders: TX {!r}, RX {!r}".format(tx_shoulder_tuple, rx_shoulder_tuple),
@@ -338,10 +346,59 @@ def engine_worker():
                     results['state'] = 'Update Predistorter'
                     results['stateprogress'] = 100
                     results['summary'] = ["Signal measurements after predistortion update"] + summary
-    finally:
-        with lock:
-            results['state'] = 'Terminated'
-            results['stateprogress'] = 0
+            elif cmd.startswith("restore_dump-"):
+                _, _, dump_id = cmd.partition("-")
+                if dump_id == "defaults":
+                    model.reset_coefs()
+                    dpddata = model.get_dpd_data()
+                    adapt.set_predistorter(dpddata)
+
+                    tx_gain = adapt.get_txgain()
+                    rx_gain = adapt.get_rxgain()
+                    digital_gain = adapt.get_digital_gain()
+                    with lock:
+                        results['state'] = 'Idle'
+                        results['stateprogress'] = 100
+                        results['summary'] = [f"Restored DPD defaults",
+                            f"Running with digital gain {digital_gain}, TX gain {tx_gain} and RX gain {rx_gain}"]
+                        results['modeldata'] = repr(dpddata)
+                else:
+                    dump_file = os.path.join(plot_path, f"adapt_{dump_id}.pkl")
+                    try:
+                        d = adapt.restore(dump_file)
+                        logging.info(f"Restore: {d}")
+                        model.set_dpd_data(d['dpddata'])
+                        with lock:
+                            results['state'] = 'Idle'
+                            results['stateprogress'] = 100
+                            results['summary'] = [f"Restored DPD settings from dumpfile {dump_id}",
+                                f"Running with digital gain {d['digital_gain']}, TX gain {d['txgain']} and RX gain {d['rxgain']}"]
+                            results['modeldata'] = repr(d["dpddata"])
+                    except:
+                        e = traceback.format_exc()
+                        with lock:
+                            results['state'] = 'Idle'
+                            results['stateprogress'] = 100
+                            results['summary'] = [f"Failed to restore DPD settings from dumpfile {dump_id}",
+                                    f"Error: {e}"]
+        except:
+            e = traceback.format_exc()
+            logging.error(e)
+            with lock:
+                results['summary'] = [f"Exception:"] + e.split("\n")
+                results['state'] = 'Autorestart pending'
+                results['stateprogress'] = 0
+
+            for i in range(5):
+                time.sleep(2)
+                with lock:
+                    results['stateprogress'] += 20
+            time.sleep(2)
+            with lock:
+                dt = datetime.datetime.utcnow().isoformat()
+                results['summary'] = [f"DPD engine auto-restarted at {dt} UTC", f"After exception {e}"]
+                results['state'] = 'Idle'
+                results['stateprogress'] = 0
 
 
 engine = Thread(target=engine_worker)
@@ -367,13 +424,10 @@ try:
             logging.info('YAML-RPC request : {}'.format(method))
             command_queue.put(method)
             cmd_socket.send_success_response(addr, msg_id, None)
-        elif method == 'set_setting':
-            logging.info('YAML-RPC request : {} -> {}'.format(method, params))
-            # params == {'setting': ..., 'value': ...}
+        elif method == 'restore_dump':
+            logging.info('YAML-RPC restore dump {}'.format(params['dump_id']))
+            command_queue.put(f"restore_dump-{params['dump_id']}")
             cmd_socket.send_success_response(addr, msg_id, None)
-        elif method == 'get_settings':
-            with lock:
-                cmd_socket.send_success_response(addr, msg_id, settings)
         elif method == 'get_results':
             with lock:
                 cmd_socket.send_success_response(addr, msg_id, results)
@@ -511,7 +565,7 @@ while i < num_iter:
 # The MIT License (MIT)
 #
 # Copyright (c) 2017 Andreas Steger
-# Copyright (c) 2018 Matthias P. Braendli
+# Copyright (c) 2019 Matthias P. Braendli
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
diff --git a/python/gui/api.py b/python/gui/api.py
index 42c89c9..c0effde 100755
--- a/python/gui/api.py
+++ b/python/gui/api.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-#   Copyright (C) 2018
+#   Copyright (C) 2019
 #   Matthias P. Braendli, matthias.braendli@mpb.li
 #
 #    http://www.opendigitalradio.org
@@ -125,12 +125,20 @@ class API:
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def dpd_settings(self, setting: str, value: str, **kwargs):
+    def dpd_restore_dump(self, **kwargs):
         if cherrypy.request.method == 'POST':
-            data = {'setting': setting, 'value': value}
-            return self._wrap_dpd("set_setting", data)
+            cl = cherrypy.request.headers['Content-Length']
+            rawbody = cherrypy.request.body.read(int(cl))
+            params = json.loads(rawbody.decode())
+            if 'dump_id' in params:
+                data = {'dump_id': params['dump_id']}
+                return self._wrap_dpd("restore_dump", data)
+            else:
+                cherrypy.response.status = 400
+                return send_error("Missing dump_id")
         else:
-            return self._wrap_dpd("get_settings")
+            cherrypy.response.status = 400
+            return send_error("POST only")
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
diff --git a/python/gui/static/js/odr-predistortion.js b/python/gui/static/js/odr-predistortion.js
index b5f29ea..4dae068 100644
--- a/python/gui/static/js/odr-predistortion.js
+++ b/python/gui/static/js/odr-predistortion.js
@@ -1,4 +1,4 @@
-//   Copyright (C) 2018
+//   Copyright (C) 2019
 //   Matthias P. Braendli, matthias.braendli@mpb.li
 //
 //    http://www.opendigitalradio.org
@@ -18,6 +18,8 @@
 //   You should have received a copy of the GNU General Public License
 //   along with ODR-DabMod.  If not, see <http://www.gnu.org/licenses/>.
 
+var adapt_dumps = [];
+
 function resultrefresh() {
     var jqxhr = doApiRequestGET("/api/dpd_results", function(data) {
         var summary = "";
@@ -51,6 +53,8 @@ function resultrefresh() {
         else {
             $('#dpdmodelplot').attr('src', "");
         }
+
+        adapt_dumps = data['adapt_dumps'];
     });
 
     jqxhr.always(function() {
@@ -58,6 +62,30 @@ function resultrefresh() {
     });
 }
 
+function adaptdumpsrefresh() {
+    $('#dpdadaptdumps').html("");
+
+    $.each(adapt_dumps, function(i, item) {
+        console.log(item);
+
+        if (isNaN(+item)) {
+            $('#dpdadaptdumps').append($('<option>', {
+                value: item,
+                text : "DPD settings from " + item,
+            }));
+        }
+        else {
+            var d = new Date(0);
+            d.setUTCSeconds(item);
+
+            $('#dpdadaptdumps').append($('<option>', {
+                value: item,
+                text : "DPD settings from " + d.toISOString(),
+            }));
+        }
+    });
+}
+
 $(function(){
     setTimeout(resultrefresh, 20);
 
@@ -86,72 +114,20 @@ $(function(){
         });
     });
 
-});
+    $('#adaptdumpsrefreshbtn').click(adaptdumpsrefresh);
 
-/*
-function calibraterefresh() {
-    doApiRequestGET("/api/calibrate", function(data) {
-        var text = "Captured TX signal and feedback." +
-            " TX median: " + data['tx_median'] +
-            " RX median: " + data['rx_median'] +
-            " with relative timestamp offset " +
-            (data['tx_ts'] - data['rx_ts']) +
-            " and measured offset " + data['coarse_offset'] +
-            ". Correlation: " + data['correlation'];
-        $('#calibrationresults').text(text);
-    });
-}
-
-$(function(){
-    $('#refreshframesbtn').click(function() {
-        var d = new Date();
-        var n = d.getTime();
-        $('#txframeimg').src = "dpd/txframe.png?cachebreak=" + n;
-        $('#rxframeimg').src = "dpd/rxframe.png?cachebreak=" + n;
-    });
+    $('#adaptdumpsload').click(function() {
+        var elt = document.getElementById("dpdadaptdumps");
 
-    $('#capturebutton').click(function() {
-        doApiRequestPOST("/api/trigger_capture", {}, function(data) {
-            console.log("trigger_capture succeeded: " + JSON.stringify(data));
-        });
-    });
-
-    $('#dpdstatusbutton').click(function() {
-        doApiRequestGET("/api/dpd_status", function(data) {
-            console.log("dpd_status succeeded: " + JSON.stringify(data));
-            $('#histogram').text(data.histogram);
-            $('#capturestatus').text(data.capture.status);
-            $('#capturelength').text(data.capture.length);
-            $('#tx_median').text(data.capture.tx_median);
-            $('#rx_median').text(data.capture.rx_median);
-        });
-
-    $.ajax({
-        type: "GET",
-        url: "/api/dpd_capture_pointcloud",
-
-        error: function(data) {
-            if (data.status == 500) {
-                var errorWindow = window.open("", "_self");
-                errorWindow.document.write(data.responseText);
-            }
-            else {
-                $.gritter.add({ title: 'API',
-                    text: "AJAX failed: " + data.statusText,
-                    image: '/fonts/warning.png',
-                    sticky: true,
-                });
-            }
-        },
-        success: function(data) {
-            $('#dpd_pointcloud').value(data)
+        if (elt.selectedIndex != -1) {
+            var selectedoption = elt.options[elt.selectedIndex].value;
+            doApiRequestPOST("/api/dpd_restore_dump", {dump_id: selectedoption}, function(data) {
+                console.log("reset succeeded: " + JSON.stringify(data));
+            });
         }
-    })
     });
 });
 
-*/
-
 
 // ToolTip init
 $(function(){
diff --git a/python/gui/templates/predistortion.html b/python/gui/templates/predistortion.html
index 8682054..62e8503 100644
--- a/python/gui/templates/predistortion.html
+++ b/python/gui/templates/predistortion.html
@@ -14,7 +14,7 @@
 
           <div class="container-fluid">
             <div class="row">
-              <div class="col-sm-8">
+              <div class="col-sm-4">
                 <h2>Current DPDCE status</h2>
                 <div>
                   <div id="dpdstatus" style="font-weight:bold;">N/A</div>
@@ -27,6 +27,21 @@
                 <div class="well well-sm" id="dpdresults">N/A</div>
               </div>
               <div class="col-sm-4">
+                <h2>List of saved DPD settings</h2>
+                <!--TODO: 'erase' and 'clear' buttons. Show DPD settings in tooltip?-->
+                <p>This list contains previously used predistortion settings that you
+                can recall.</p>
+                <p>
+                <select id="dpdadaptdumps" size="8" style="width:70%" multiple></select>
+                </p>
+                <p>
+                <button type="button" class="btn btn-sm btn-info" id="adaptdumpsrefreshbtn">Refresh
+                </button>
+                <button type="button" class="btn btn-sm btn-warning" id="adaptdumpsload">Load and Apply
+                </button>
+                </p>
+              </div>
+              <div class="col-sm-4">
                 <h2>Summary</h2>
                 <p>Calibration needs to be done once before the PA model
                 can be trained. Every time calibration is changed, the predistortion
@@ -36,6 +51,8 @@
                 runs are needed before the model can be trained.</p>
                 <p>The capture and model analysis will calculate a new set of
                 DPD model data, that you can apply using the Update Predistorter button.</p>
+                <p>The reset button allows you to reset the computation engine. It does not
+                modify the currently active predistorter.</p>
               </div>
             </div>
           </div>
@@ -47,7 +64,7 @@
           <button type="button" class="btn btn-sm btn-warning" id="adaptbtn">
             Update Predistorter</button>
           <button type="button" class="btn btn-sm btn-info" id="resetbtn">
-            Reset captured data</button>
+            Reset Capture and Model</button>
         </div>
       </div>
 
@@ -97,7 +114,7 @@
 </html>
 
 <!--
-   Copyright (C) 2018
+   Copyright (C) 2019
    Matthias P. Braendli, matthias.braendli@mpb.li
 
 This file is part of ODR-DabMod.