GUI: add capture and plot to DPD

6 files changed, 134 insertions, 38 deletions

diff --git a/python/dpd/ExtractStatistic.py b/python/dpd/ExtractStatistic.py
index 639513a..1aa4391 100644
--- a/python/dpd/ExtractStatistic.py
+++ b/python/dpd/ExtractStatistic.py
@@ -41,6 +41,8 @@ class ExtractStatistic:
     def __init__(self, c):
         self.c = c
 
+        self._plot_data = None
+
         # Number of measurements used to extract the statistic
         self.n_meas = 0
 
@@ -58,12 +60,10 @@ class ExtractStatistic:
         for i in range(c.ES_n_bins):
             self.tx_values_lists.append([])
 
-        self.plot = c.ES_plot
+    def plot(self, plot_path, title):
+        if self._plot_data is not None:
+            tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists = self._plot_data
 
-    def _plot_and_log(self, tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists):
-        if self.plot and self.c.plot_location is not None:
-            dt = datetime.datetime.now().isoformat()
-            fig_path = self.c.plot_location + "/" + dt + "_ExtractStatistic.png"
             sub_rows = 3
             sub_cols = 1
             fig = plt.figure(figsize=(sub_cols * 6, sub_rows / 2. * 6))
@@ -80,7 +80,7 @@ class ExtractStatistic:
                            np.abs(rx_values_list),
                            s=0.1,
                            color="black")
-            ax.set_title("Extracted Statistic")
+            ax.set_title("Extracted Statistic {}".format(title))
             ax.set_xlabel("TX Amplitude")
             ax.set_ylabel("RX Amplitude")
             ax.set_ylim(0, 0.8)
@@ -116,7 +116,7 @@ class ExtractStatistic:
             ax.set_ylim(0, self.n_per_bin * 1.2)
 
             fig.tight_layout()
-            fig.savefig(fig_path)
+            fig.savefig(plot_path)
             plt.close(fig)
 
     def _rx_value_per_bin(self):
@@ -166,7 +166,7 @@ class ExtractStatistic:
         phase_diffs_values_lists = self._phase_diff_list_per_bin()
         phase_diffs_values = _phase_diff_value_per_bin(phase_diffs_values_lists)
 
-        self._plot_and_log(tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists)
+        self._plot_data = (tx_values, rx_values, phase_diffs_values, phase_diffs_values_lists)
 
         tx_values_crop = np.array(tx_values, dtype=np.float32)[:idx_end]
         rx_values_crop = np.array(rx_values, dtype=np.float32)[:idx_end]
@@ -176,6 +176,7 @@ class ExtractStatistic:
 # The MIT License (MIT)
 #
 # Copyright (c) 2017 Andreas Steger
+# Copyright (c) 2018 Matthias P. Braendli
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
diff --git a/python/dpd/GlobalConfig.py b/python/dpd/GlobalConfig.py
index 873b6ac..99280f2 100644
--- a/python/dpd/GlobalConfig.py
+++ b/python/dpd/GlobalConfig.py
@@ -10,7 +10,7 @@
 import numpy as np
 
 class GlobalConfig:
-    def __init__(self, samplerate, plot_location: str):
+    def __init__(self, samplerate: int, plot_location: str):
         self.sample_rate = samplerate
         assert self.sample_rate == 8192000, "We only support constants for 8192000 sample rate: {}".format(self.sample_rate)
 
diff --git a/python/dpdce.py b/python/dpdce.py
index 379f3d0..a9ed140 100755
--- a/python/dpdce.py
+++ b/python/dpdce.py
@@ -43,7 +43,8 @@ rc_port = config.getint('rc_port')
 samplerate = config.getint('samplerate')
 samps = config.getint('samps')
 coef_file = config['coef_file']
-log_folder = config['log_folder']
+logs_directory = config['logs_directory']
+plot_directory = config['plot_directory']
 
 import logging
 import datetime
@@ -52,7 +53,7 @@ save_logs = False
 
 # Simple usage scenarios don't need to clutter /tmp
 if save_logs:
-    dt = datetime.datetime.now().isoformat()
+    dt = datetime.datetime.utcnow().isoformat()
     logging_path = '/tmp/dpd_{}'.format(dt).replace('.', '_').replace(':', '-')
     print("Logs and plots written to {}".format(logging_path))
     os.makedirs(logging_path)
@@ -71,7 +72,7 @@ if save_logs:
     # add the handler to the root logger
     logging.getLogger('').addHandler(console)
 else:
-    dt = datetime.datetime.now().isoformat()
+    dt = datetime.datetime.utcnow().isoformat()
     logging.basicConfig(format='%(asctime)s - %(module)s - %(levelname)s - %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S',
                         level=logging.INFO)
@@ -83,6 +84,7 @@ import socket
 from lib import yamlrpc
 import numpy as np
 import traceback
+import os.path
 from threading import Thread, Lock
 from queue import Queue
 from dpd.Model import Poly
@@ -96,13 +98,14 @@ from dpd.GlobalConfig import GlobalConfig
 from dpd.MER import MER
 from dpd.Measure_Shoulders import Measure_Shoulders
 
-c = GlobalConfig(samplerate, logging_path)
+plot_path = os.path.realpath(plot_directory)
+
+c = GlobalConfig(samplerate, plot_path)
 symbol_align = Symbol_align(c)
 mer = MER(c)
 meas_shoulders = Measure_Shoulders(c)
 meas = Measure(c, samplerate, dpd_port, samps)
-extStat = ExtractStatistic(c)
-adapt = Adapt(rc_port, coef_file, logging_path)
+adapt = Adapt(rc_port, coef_file, plot_path)
 
 model = Poly(c)
 
@@ -134,7 +137,11 @@ settings = {
         'digital_gain': digital_gain,
         'dpddata': dpddata,
         }
+internal_data = {
+        'n_runs': 0,
+        }
 results = {
+        'statplot': None,
         'tx_median': 0,
         'rx_median': 0,
         'state': 'Idle',
@@ -148,6 +155,7 @@ command_queue = Queue(maxsize=1)
 agc = Agc(meas, adapt, c)
 
 def engine_worker():
+    extStat = ExtractStatistic(c)
     try:
         while True:
             cmd = command_queue.get()
@@ -180,8 +188,86 @@ def engine_worker():
                     results['tx_median'] = float(tx_median)
                     results['rx_median'] = float(rx_median)
                     results['state'] = 'Idle'
-                    results['stateprogress'] = 0
+                    results['stateprogress'] = 100
                     results['summary'] = ["Calibration was done:"] + summary
+            elif cmd == "reset":
+                with lock:
+                    internal_data['n_runs'] = 0
+                    results['state'] = 'Idle'
+                    results['stateprogress'] = 0
+                    results['summary'] = ["Reset"]
+                extStat = ExtractStatistic(c)
+            elif cmd == "trigger_run":
+                with lock:
+                    results['state'] = 'Capture + Model'
+                    results['stateprogress'] = 0
+                    n_runs = internal_data['n_runs']
+
+                # Get Samples and check gain
+                txframe_aligned, tx_ts, rxframe_aligned, rx_ts, rx_median, tx_median = meas.get_samples()
+                # TODO Check TX median
+
+                with lock:
+                    results['stateprogress'] = 20
+                    results['summary'] = ["Captured {} samples".format(len(txframe_aligned)),
+                        "TX/RX median: {} / {}".format(tx_median, rx_median)]
+
+                # Extract usable data from measurement
+                tx, rx, phase_diff, n_per_bin = extStat.extract(txframe_aligned, rxframe_aligned)
+
+                time = datetime.datetime.utcnow()
+
+                plot_file = "stats_{}.png".format(time.strftime("%s"))
+                extStat.plot(os.path.join(plot_path, plot_file), time.strftime("%Y-%m-%dT%H%M%S"))
+
+                with lock:
+                    results['statplot'] = "dpd/" + plot_file
+                    results['stateprogress'] = 30
+                    results['summary'] += ["Extracted Statistics".format(tx_median, rx_median)]
+
+                n_meas = Heuristics.get_n_meas(n_runs)
+                if extStat.n_meas >= n_meas:  # Use as many measurements nr of runs
+                    if any(x is None for x in [tx, rx, phase_diff]):
+                        with lock:
+                            results['summary'] += ["Error! No data to calculate model"]
+                            results['state'] = 'Idle'
+                            results['stateprogress'] = 0
+                    else:
+                        with lock:
+                            results['state'] = 'Capture + Model'
+                            results['stateprogress'] = 40
+                            results['summary'] += ["Training model"]
+
+                        model.train(tx, rx, phase_diff, lr=Heuristics.get_learning_rate(n_runs))
+
+                        with lock:
+                            results['state'] = 'Capture + Model'
+                            results['stateprogress'] = 60
+                            results['summary'] += ["Getting DPD data"]
+
+                        dpddata = model.get_dpd_data()
+                        with lock:
+                            internal_data['dpddata'] = dpddata
+                            internal_data['n_runs'] = 0
+
+                            results['state'] = 'Capture + Model'
+                            results['stateprogress'] = 80
+                            results['summary'] += ["Reset statistics"]
+
+                        extStat = ExtractStatistic(c)
+
+                        with lock:
+                            results['state'] = 'Idle'
+                            results['stateprogress'] = 100
+                            results['summary'] += ["New DPD coefficients calculated"]
+
+                    with lock:
+                        internal_data['n_runs'] += 1
+                else:
+                    with lock:
+                        results['state'] = 'Idle'
+                        results['stateprogress'] = 100
+                        results['summary'] += ["More data required to train model"]
 
     finally:
         with lock:
diff --git a/python/gui-dpdce.ini b/python/gui-dpdce.ini
index 48c6abf..4385c80 100644
--- a/python/gui-dpdce.ini
+++ b/python/gui-dpdce.ini
@@ -19,7 +19,10 @@ samps=81920
 coef_file=poly.coef
 
 # Write logs to this folder, or leave empty for no logs
-log_folder=
+logs_directory=
+
+# Saving plots to the static directory makes them accessible to the browser
+plot_directory=gui/static/dpd
 
 
 [gui]
diff --git a/python/gui/static/js/odr-predistortion.js b/python/gui/static/js/odr-predistortion.js
index 739e0ee..e9f7c96 100644
--- a/python/gui/static/js/odr-predistortion.js
+++ b/python/gui/static/js/odr-predistortion.js
@@ -35,6 +35,10 @@ function resultrefresh() {
         }
         $('#dpdprogress').css('width', percentage + '%');
         $('#dpdprogresstext').text(percentage + '%');
+
+        if (data['statplot']) {
+            $('#dpdcapturestats').attr('src', data['statplot']);
+        }
     });
 
     jqxhr.always(function() {
@@ -43,7 +47,7 @@ function resultrefresh() {
 }
 
 $(function(){
-    setTimeout(resultrefresh, 2000);
+    setTimeout(resultrefresh, 20);
 
     $('#calibratebtn').click(function() {
         doApiRequestPOST("/api/dpd_calibrate", {}, function(data) {
@@ -51,6 +55,17 @@ $(function(){
         });
     });
 
+    $('#resetbtn').click(function() {
+        doApiRequestPOST("/api/dpd_reset", {}, function(data) {
+            console.log("reset succeeded: " + JSON.stringify(data));
+        });
+    });
+
+    $('#triggerbtn').click(function() {
+        doApiRequestPOST("/api/dpd_trigger_run", {}, function(data) {
+            console.log("run succeeded: " + JSON.stringify(data));
+        });
+    });
 });
 
 /*
diff --git a/python/gui/templates/predistortion.html b/python/gui/templates/predistortion.html
index edb2f04..adbba7e 100644
--- a/python/gui/templates/predistortion.html
+++ b/python/gui/templates/predistortion.html
@@ -26,35 +26,26 @@
           <div>Calibration needs to be done once before the PA model
           can be trained. Every time calibration is changed, the predistortion
           parameters are invalidated!</div>
+          <div>Once calibration succeeded and correct RX gain is set, you
+            can trigger a capture and model the PA. Usually, several capture
+            runs are needed before the model can be trained.</div>
           <button type="button" class="btn btn-sm btn-warning" id="calibratebtn">
             Calibrate</button>
-        </div>
-      </div>
-      <!--
-      <div class="panel panel-default">
-        <div class="panel-heading">Capture TX and RX frames</div>
-        <div class="panel-body">
-          <div>
-            <img id="txframeimg" src="dpd/txframe.png" width="320" height="240" />
-            <img id="rxframeimg" src="dpd/rxframe.png" width="320" height="240" />
-          </div>
-          <div>
-            <button type="button" class="btn btn-sm btn-info" id="refreshframesbtn">
-              Refresh</button>
-          </div>
+          <button type="button" class="btn btn-sm btn-warning" id="triggerbtn">
+            Trigger Capture and PA Modeling</button>
+          <button type="button" class="btn btn-sm btn-info" id="resetbtn">
+            Reset captured data</button>
         </div>
       </div>
 
       <div class="panel panel-default">
-        <div class="panel-heading">Capture</div>
+        <div class="panel-heading">Capture Statistics</div>
         <div class="panel-body">
-          <div>On pressing this button,
-            the DPDCE will trigger a capture and a quick data
-            analysis, without updating any DPD models.</div>
-          <button type="button" class="btn btn-sm btn-info" id="capturebutton">
-            Capture</button>
+          <img id="dpdcapturestats" />
         </div>
       </div>
+
+      <!--
       <div class="panel panel-default">
         <div class="panel-heading">Status</div>
         <div class="panel-body">