Rework GUI and DPDCE

29 files changed, 23 insertions, 427 deletions

diff --git a/python/dpd/src/Adapt.py b/python/dpd/Adapt.py
index a57602f..840aee9 100644
--- a/python/dpd/src/Adapt.py
+++ b/python/dpd/Adapt.py
@@ -66,11 +66,11 @@ class Adapt:
         ZMQ remote control.
     """
 
-    def __init__(self, config, port, coef_path):
+    def __init__(self, port, coef_path, plot_location):
         logging.debug("Instantiate Adapt object")
-        self.c = config
         self.port = port
         self.coef_path = coef_path
+        self.plot_location = plot_location
         self.host = "localhost"
         self._context = zmq.Context()
 
@@ -238,8 +238,8 @@ class Adapt:
         """Backup current settings to a file"""
         dt = datetime.datetime.now().isoformat()
         if path is None:
-            if self.c.plot_location is not None:
-                path = self.c.plot_location + "/" + dt + "_adapt.pkl"
+            if self.plot_location is not None:
+                path = self.plot_location + "/" + dt + "_adapt.pkl"
             else:
                 raise Exception("Cannot dump Adapt without either plot_location or path set")
         d = {
diff --git a/python/dpd/src/Dab_Util.py b/python/dpd/Dab_Util.py
index bc89a39..f9ae59f 100644
--- a/python/dpd/src/Dab_Util.py
+++ b/python/dpd/Dab_Util.py
@@ -13,8 +13,8 @@ import matplotlib
 
 matplotlib.use('agg')
 import matplotlib.pyplot as plt
-import src.subsample_align as sa
-import src.phase_align as pa
+import dpd.subsample_align as sa
+import dpd.phase_align as pa
 from scipy import signal
 
 
diff --git a/python/dpd/src/ExtractStatistic.py b/python/dpd/ExtractStatistic.py
index 639513a..639513a 100644
--- a/python/dpd/src/ExtractStatistic.py
+++ b/python/dpd/ExtractStatistic.py
diff --git a/python/dpd/src/GlobalConfig.py b/python/dpd/GlobalConfig.py
index 56839fc..abd7442 100644
--- a/python/dpd/src/GlobalConfig.py
+++ b/python/dpd/GlobalConfig.py
@@ -10,11 +10,12 @@
 import numpy as np
 
 class GlobalConfig:
-    def __init__(self, cli_args, plot_location):
-        self.sample_rate = cli_args.samplerate
+    def __init__(self, args, plot_location: str):
+        self.sample_rate = args['samplerate']
         assert self.sample_rate == 8192000  # By now only constants for 8192000
 
         self.plot_location = plot_location
+        plot = len(plot_location) > 0
 
         # DAB frame
         # Time domain
@@ -39,7 +40,7 @@ class GlobalConfig:
         self.phase_offset_per_sample = 1. / self.sample_rate * 2 * np.pi * 1000
 
         # Constants for ExtractStatistic
-        self.ES_plot = cli_args.plot
+        self.ES_plot = plot
         self.ES_start = 0.0
         self.ES_end = 1.0
         self.ES_n_bins = 64  # Number of bins between ES_start and ES_end
@@ -47,7 +48,7 @@ class GlobalConfig:
 
         # Constants for Measure_Shoulder
         self.MS_enable = False
-        self.MS_plot = cli_args.plot
+        self.MS_plot = plot
 
         meas_offset = 976  # Offset from center frequency to measure shoulder [kHz]
         meas_width = 100  # Size of frequency delta to measure shoulder [kHz]
@@ -65,24 +66,18 @@ class GlobalConfig:
         self.MS_n_proc = 4
 
         # Constants for MER
-        self.MER_plot = cli_args.plot
+        self.MER_plot = plot
 
         # Constants for Model
-        self.MDL_plot = cli_args.plot
+        self.MDL_plot = plot
 
         # Constants for Model_PM
         # Set all phase offsets to zero for TX amplitude < MPM_tx_min
         self.MPM_tx_min = 0.1
 
-        # Constants for TX_Agc
-        self.TAGC_max_txgain = 89  # USRP B200 specific
-        self.TAGC_tx_median_target = cli_args.target_median
-        self.TAGC_tx_median_max = self.TAGC_tx_median_target * 1.4
-        self.TAGC_tx_median_min = self.TAGC_tx_median_target / 1.4
-
         # Constants for RX_AGC
         self.RAGC_min_rxgain = 25  # USRP B200 specific
-        self.RAGC_rx_median_target = cli_args.target_median
+        self.RAGC_rx_median_target = 0.05
 
 # The MIT License (MIT)
 #
diff --git a/python/dpd/src/Heuristics.py b/python/dpd/Heuristics.py
index 21d400b..21d400b 100644
--- a/python/dpd/src/Heuristics.py
+++ b/python/dpd/Heuristics.py
diff --git a/python/dpd/src/MER.py b/python/dpd/MER.py
index 693058d..693058d 100644
--- a/python/dpd/src/MER.py
+++ b/python/dpd/MER.py
diff --git a/python/dpd/src/Measure.py b/python/dpd/Measure.py
index 6d8007d..36b1888 100644
--- a/python/dpd/src/Measure.py
+++ b/python/dpd/Measure.py
@@ -9,7 +9,7 @@
 import socket
 import struct
 import numpy as np
-import src.Dab_Util as DU
+import dpd.Dab_Util as DU
 import os
 import logging
 
diff --git a/python/dpd/src/Measure_Shoulders.py b/python/dpd/Measure_Shoulders.py
index fd90050..fd90050 100644
--- a/python/dpd/src/Measure_Shoulders.py
+++ b/python/dpd/Measure_Shoulders.py
diff --git a/python/dpd/src/Model.py b/python/dpd/Model.py
index b2c303f..a8aedde 100644
--- a/python/dpd/src/Model.py
+++ b/python/dpd/Model.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from src.Model_Poly import Poly
-from src.Model_Lut import Lut
+from dpd.Model_Poly import Poly
+from dpd.Model_Lut import Lut
 
 def select_model_from_dpddata(dpddata):
     if dpddata[0] == 'lut':
diff --git a/python/dpd/src/Model_AM.py b/python/dpd/Model_AM.py
index 75b226f..75b226f 100644
--- a/python/dpd/src/Model_AM.py
+++ b/python/dpd/Model_AM.py
diff --git a/python/dpd/src/Model_Lut.py b/python/dpd/Model_Lut.py
index e70fdb0..e70fdb0 100644
--- a/python/dpd/src/Model_Lut.py
+++ b/python/dpd/Model_Lut.py
diff --git a/python/dpd/src/Model_PM.py b/python/dpd/Model_PM.py
index 7b80bf3..7b80bf3 100644
--- a/python/dpd/src/Model_PM.py
+++ b/python/dpd/Model_PM.py
diff --git a/python/dpd/src/Model_Poly.py b/python/dpd/Model_Poly.py
index cdfd319..c8f6135 100644
--- a/python/dpd/src/Model_Poly.py
+++ b/python/dpd/Model_Poly.py
@@ -9,8 +9,8 @@ import os
 import logging
 import numpy as np
 
-import src.Model_AM as Model_AM
-import src.Model_PM as Model_PM
+import dpd.Model_AM as Model_AM
+import dpd.Model_PM as Model_PM
 
 
 def assert_np_float32(x):
diff --git a/python/dpd/README.md b/python/dpd/README.md
deleted file mode 100644
index 307a2f5..0000000
--- a/python/dpd/README.md
+++ /dev/null
@@ -1,264 +0,0 @@
-Digital Predistortion Computation Engine for ODR-DabMod
-=======================================================
-
-This folder contains a digital predistortion prototype.
-It was only tested in a laboratory system, and is not ready
-for production usage.
-
-Concept
--------
-
-ODR-DabMod makes outgoing TX samples and feedback RX samples available to an
-external tool. This external tool can request a buffer of samples for analysis,
-can calculate coefficients for the predistorter in ODR-DabMod and load the new
-coefficients using the remote control.
-
-The external tool is called the Digital Predistortion Computation Engine (DPDCE).
-The DPDCE is written in python, and makes use of the numpy library for
-efficient computation. Its sources reside in the *dpd* folder.
-
-The predistorter in ODR-DabMod supports two modes: polynomial and lookup table.
-In the DPDCE, only the polynomial model is implemented at the moment.
-
-The *dpd/main.py* script is the entry point for the *DPD Computation Engine*
-into which these features will be implemented. The tool uses modules from the
-*dpd/src/* folder:
-
-- Sample transfer and time alignment with subsample accuracy is done by *Measure.py*
-- Estimating the effects of the PA using some model and calculation of the updated
-  polynomial coefficients is done in *Model.py* and other specific *Model_XXX.py* files
-- Finally, *Adapt.py* updates the ODR-DabMod predistortion setting and digital gain
-
-These modules themselves use additional helper scripts in the *dpd/src/* folder.
-
-Requirements
-------------
-
-- USRP B200.
-- Power amplifier.
-- A feedback connection from the power amplifier output, such that the average power level at
-  the USRP RX port is at -45dBm or lower.
-  Usually this is done with a directional coupler and additional attenuators.
-- ODR-DabMod with enabled *dpd_port*, and with a samplerate of 8192000 samples per second.
-- Synchronous=1 so that the USRP has the timestamping set properly, internal refclk and pps
-  are sufficient (not GPSDO necessary).
-- A live mux source with TIST enabled.
-
-See dpd/dpd.ini for an example.
-
-The DPD server port can be tested with the *dpd/show_spectrum.py* helper tool, which can also display
-a constellation diagram.
-
-Hardware Setup
---------------
-
-![setup diagram](img/setup_diagram.svg)
-![setup photo](img/setup_photo.svg)
-
-Our setup is depicted in the Figure above. We used components with the following properties:
- 1. USRP TX (max +20dBm)
- 2. Band III Filter (Mini-Circuits RBP-220W+, 190-250MHz, -3.5dB)
- 3. Power amplifier (Mini-Circuits, max +15dBm output, +10 dB gain at 200MHz)
- 4. Directional coupler (approx. -25dB @ 223MHz)
- 5. Attenuator (-20 dB)
- 6. Attenuator (-30 dB)
- 7. USRP RX (max -15dBm input allowed, max -45dBm desired)
- 8. Spectrum analyzer (max +30dBm allowed)
-
-It is important to make sure that the USRP RX port does not receive too much
-power. Otherwise the USRP will break. Here is an example of how we calculated
-the maximal USRP RX input power for our case. As this is only a rough
-calculation to protect the port, the predistortion software will later
-automatically apply a normalization for the RX input by adapting the USRP RX
-gain.
-
-    TX Power + PA Gain - Coupling Factor - Attenuation = 20dBm + 10dB -25dB -50dB = -45dBm
-
-Thus we have a margin of about 30dB for the input power of the USRP RX port.
-Keep in mind we need to calculate using peak power, not average power, and it is
-essential that there is no nonlinearity in the RX path!
-
-Software Setup
---------------
-
-We assume that you already installed *ODR-DabMux* and *ODR-DabMod*.
-You should install the required python dependencies for the DPDCE using
-distribution packages. You will need at least scipy, matplotlib and
-python-zeromq.
-
-Use the predistortion
-----------------------
-
-Make sure you have a ODR-DabMux running with a TCP output on port 9200.
-
-Then run the modulator, with the example dpd configuration file.
-
-```
-./odr-dabmod dpd/dpd.ini
-```
-
-This configuration file is different from usual defaults in several respects:
-
- * logging to /tmp/dabmod.log
- * 4x oversampling: 8192000 sample rate
- * a very small digital gain, which will be overridden by the DPDCE
- * predistorter enabled
-
-The TX gain should be chosen so that you can drive your amplifier into
-saturation with a digital gain of 0.1, so that there is margin for the DPD to
-operate.
-
-You should *not modify txgain, rxgain, digital gain or coefficient settings manually!*
-When the DPDCE is used, it controls these settings, and there are command line
-options for you to define initial values.
-
-To verify that the communication between the DPDCE and ODR-DabMod is ok,
-you can use the status and reset options:
-
-```
-cd dpd
-python main.py --status
-python main.py --reset
-```
-
-The reset option sets all DPD-related settings to the defaults (as shown in the
-`--help` usage screen) and stops.
-
-When neither `--status` nor `--reset` is given, the DPDCE will run the predistortion
-algorithm. As a first test you should run the DPDCE with the `--plot`
-parameter. It preserves the output power and generates all available
-visualisation plots in the newly created logging directory
-`/tmp/dpd_<time_stamp>`. As the predistortion should increase the peak to
-shoulder ratio, you should select a *txgain* in the ODR-DabMod configuration
-file such that the initial peak-to-soulder ratio visible on your spectrum
-analyser. This way, you will be able to see a the
-change.
-
-```
-cd dpd
-python main.py --plot
-```
-
-The DPDCE now does 10 iterations, and tries to improve the predistortion effectiveness.
-In each step the learning rate is decreased. The learning rate is the factor
-with which new coefficients are weighted in a weighted mean with the old
-coefficients. Moreover the nuber of measurements increases in each iteration.
-You find more information about that in *Heuristic.py*.
-
-Each plot is stored to the logging directory under a filename containing its
-time stamp and its label. Following plots are generated chronologically:
-
- - ExtractStatistic: Extracted information from one or multiple measurements.
- - Model\_AM: Fitted function for the amplitudes of the power amplifier against the TX amplitude.
- - Model\_PM: Fitted function for the phase difference of the power amplifier against the TX amplitude.
- - adapt.pkl: Contains all settings for the predistortion.
-   You can load them again without doing measurements with the `apply_adapt_dumps.py` script.
- - MER: Constellation diagram used to calculate the modulation error rate.
-
-After the run you should be able to observe that the peak-to-shoulder
-difference has increased on your spectrum analyzer, similar to the figure below.
-
-Without digital predistortion:
-
-![shoulder_measurement_before](img/shoulder_measurement_before.png)
-
-With digital predistortion, computed by the DPDCE:
-
-![shoulder_measurement_after](img/shoulder_measurement_after.png)
-
-Now see what happens if you apply the predistortions for different TX gains.
-You can either set the TX gain before you start the predistortion or using the
-command line option `--txgain gain`. You can also try to adjust other
-parameters. To see their documentation run `python main.py --help`.
-
-File format for coefficients
-----------------------------
-The coef file contains the polynomial coefficients used in the predistorter.
-The file format is very similar to the filtertaps file used in the FIR filter.
-It is a text-based format that can easily be inspected and edited in a text
-editor.
-
-The first line contains an integer that defines the predistorter to be used:
-1 for polynomial, 2 for lookup table.
-
-For the polynomial, the subsequent line contains the number of coefficients
-as an integer. The second and third lines contain the real, respectively the
-imaginary parts of the first coefficient. Fourth and fifth lines give the
-second coefficient, and so on. The file therefore contains 1 + 1 + 2xN lines if
-it contains N coefficients.
-
-For the lookup table, the subsequent line contains a float scalefactor that is
-applied to the samples in order to bring them into the range of 32-bit unsigned
-integer. Then, the next pair of lines contains real and imaginary part of the first
-lookup-table entry, which is multiplied to samples in first range. Then it's
-followed by 31 other pairs. The entries are complex values close to 1 + 0j.
-The file therefore contains 1 + 1 + 2xN lines if it contains N coefficients.
-
-TODO
-----
-
- - Understand and fix occasional ODR-DabMod crashes when using DPDCE.
- - Make the predistortion more robust. At the moment the shoulders sometimes
-   increase instead of decrease after applying newly calculated predistortion
-   parameters. Can this behaviour be predicted from the measurement? This would
-   make it possible to filter out bad predistortion settings.
- - Find a better measurement for the quality of the predistortion. The USRP
-   might not be good enough to measure large peak-to-shoulder ratios, because
-   the ADC has 12 bits and DAB signals have a large crest factor.
- - Implement a Volterra polynomial to model the PA. Compared to the current
-   model this would also capture the time dependent behaviour of the PA (memory
-   effects).
- - Continuously observe DAB signal in frequency domain and make sure the power
-   stays the same. At the moment only the power in the time domain is kept the
-   same.
- - At the moment we assume that the USRP RX gain has to be larger than 30dB and
-   the received signal should have a median absolute value of 0.05 in order to
-   have a high quality quantization. Do measurements to support or improve
-   this heuristic.
- - Check if we need to measure MER differently (average over more symbols?)
- - Is -45dBm the best RX feedback power level?
-
-REFERENCES
-----------
-
-Some papers:
-
-The paper Raich, Qian, Zhou, "Orthogonal Polynomials for Power Amplifier
-Modeling and Predistorter Design" proposes other base polynomials that have
-less numerical instability.
-
-Aladrén, Garcia, Carro, de Mingo, and Sanchez-Perez, "Digital Predistortion
-Based on Zernike Polynomial Functions for RF Nonlinear Power Amplifiers".
-
-Jiang and Wilford, "Digital predistortion for power amplifiers using separable functions"
-
-Changsoo Eun and Edward J. Powers, "A New Volterra Predistorter Based on the Indirect Learning Architecture"
-
-Raviv Raich, Hua Qian, and G. Tong Zhou, "Orthogonal Polynomials for Power Amplifier Modeling and Predistorter Design"
-
-
-Models without memory:
-
-Complex polynomial: y[i] = a1 x[i] + a2 x[i]^2 + a3 x[i]^3 + ...
-
-The complex polynomial corresponds to the input/output relationship that
-applies to the PA in passband (real-valued signal). According to several
-sources, this gets transformed to another representation if we consider complex
-baseband instead. In the following, all variables are complex.
-
-Odd-order baseband: y[i] = (b1 + b2 abs(x[i])^2 + b3 abs(x[i])^4) + ...) x[i]
-
-Complete baseband: y[i] = (b1 + b2 abs(x[i]) + b3 abs(x[i])^2) + ...) x[i]
-
-with
-    b_k = 2^{1-k} \binom{k}{(k-1)/2} a_k
-
-
-Models with memory:
-
- - Hammerstein model: Nonlinearity followed by LTI filter
- - Wiener model: LTI filter followed by NL
- - Parallel Wiener: input goes to N delays, each delay goes to a NL, all NL outputs summed.
-
-Taken from slide 36 of [ECE218C Lecture 15](http://www.ece.ucsb.edu/Faculty/rodwell/Classes/ece218c/notes/Lecture15_Digital%20Predistortion_and_Future%20Challenges.pdf)
-
diff --git a/python/dpd/src/RX_Agc.py b/python/dpd/RX_Agc.py
index f778dee..0cc18b8 100644
--- a/python/dpd/src/RX_Agc.py
+++ b/python/dpd/RX_Agc.py
@@ -14,8 +14,8 @@ import matplotlib
 matplotlib.use('agg')
 import matplotlib.pyplot as plt
 
-import src.Adapt as Adapt
-import src.Measure as Measure
+import dpd.Adapt as Adapt
+import dpd.Measure as Measure
 
 class Agc:
     """The goal of the automatic gain control is to set the 
@@ -59,7 +59,7 @@ class Agc:
                 self.rxgain, self.min_rxgain))
 
             logging.info("RX Median {:1.4f}, estimated peak {:1.4f}, correction factor {:1.4f}, new RX gain {:1.4f}".format(
-            rx_median, rx_peak, correction_factor, self.rxgain
+                rx_median, rx_peak, correction_factor, self.rxgain
             ))
 
             self.adapt.set_rxgain(self.rxgain)
diff --git a/python/dpd/src/Symbol_align.py b/python/dpd/Symbol_align.py
index 2a17a65..2a17a65 100644
--- a/python/dpd/src/Symbol_align.py
+++ b/python/dpd/Symbol_align.py
diff --git a/python/dpd/src/TX_Agc.py b/python/dpd/TX_Agc.py
index 309193d..309193d 100644
--- a/python/dpd/src/TX_Agc.py
+++ b/python/dpd/TX_Agc.py
diff --git a/python/dpd/__init__.py b/python/dpd/__init__.py
new file mode 100644
index 0000000..9c33361
--- /dev/null
+++ b/python/dpd/__init__.py
@@ -0,0 +1 @@
+# DPDCE main module
diff --git a/python/dpd/dpd.ini b/python/dpd/dpd.ini
deleted file mode 100644
index 31d6140..0000000
--- a/python/dpd/dpd.ini
+++ /dev/null
@@ -1,60 +0,0 @@
-[remotecontrol]
-telnet=1
-telnetport=2121
-zmqctrl=1
-zmqctrlendpoint=tcp://127.0.0.1:9400
-
-[log]
-syslog=0
-filelog=1
-filename=/tmp/dabmod.log
-
-[input]
-transport=tcp
-source=localhost:9200
-
-[modulator]
-gainmode=var
-rate=8192000
-
-# keep in mind that the DPDCE will set the digital gain through the RC!
-digital_gain=0.001
-
-[firfilter]
-enabled=1
-
-[poly]
-enabled=1
-polycoeffile=dpd/poly.coef
-
-# How many threads to use for the predistorter.
-# If not set, detect automatically.
-#num_threads=2
-
-[output]
-# to prepare a file for the dpd/iq_file_server.py script,
-# use output=file
-output=uhd
-
-[fileoutput]
-filename=dpd.iq
-
-[uhdoutput]
-device=
-master_clock_rate=32768000
-type=b200
-txgain=80
-channel=13C
-refclk_source=internal
-pps_source=none
-behaviour_refclk_lock_lost=ignore
-max_gps_holdover_time=600
-dpd_port=50055
-rxgain=15
-
-[delaymanagement]
-; Use synchronous=1 so that the USRP time is set. This works
-; even in the absence of a reference clk and PPS
-synchronous=1
-mutenotimestamps=1
-offset=4.0
diff --git a/python/dpd/lut.coef b/python/dpd/lut.coef
deleted file mode 100644
index a198d56..0000000
--- a/python/dpd/lut.coef
+++ /dev/null
@@ -1,64 +0,0 @@
-2
-4294967296
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
-1
-0
diff --git a/python/dpd/apply_adapt_dumps.py b/python/dpd/old/apply_adapt_dumps.py
index 20bc013..20bc013 100755
--- a/python/dpd/apply_adapt_dumps.py
+++ b/python/dpd/old/apply_adapt_dumps.py
diff --git a/python/dpd/iq_file_server.py b/python/dpd/old/iq_file_server.py
index 7a4e570..7a4e570 100755
--- a/python/dpd/iq_file_server.py
+++ b/python/dpd/old/iq_file_server.py
diff --git a/python/dpd/main.py b/python/dpd/old/main.py
index 9ea5a39..9ea5a39 100755
--- a/python/dpd/main.py
+++ b/python/dpd/old/main.py
diff --git a/python/dpd/show_spectrum.py b/python/dpd/old/show_spectrum.py
index f23dba2..f23dba2 100755
--- a/python/dpd/show_spectrum.py
+++ b/python/dpd/old/show_spectrum.py
diff --git a/python/dpd/store_received.py b/python/dpd/old/store_received.py
index 19b735e..19b735e 100755
--- a/python/dpd/store_received.py
+++ b/python/dpd/old/store_received.py
diff --git a/python/dpd/src/phase_align.py b/python/dpd/phase_align.py
index 8654333..8654333 100644
--- a/python/dpd/src/phase_align.py
+++ b/python/dpd/phase_align.py
diff --git a/python/dpd/poly.coef b/python/dpd/poly.coef
deleted file mode 100644
index 248d316..0000000
--- a/python/dpd/poly.coef
+++ /dev/null
@@ -1,12 +0,0 @@
-1
-5
-1.0
-0.0
-0.0
-0.0
-0.0
-0.0
-0.0
-0.0
-0.0
-0.0
diff --git a/python/dpd/src/__init__.py b/python/dpd/src/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/python/dpd/src/__init__.py
+++ /dev/null
diff --git a/python/dpd/src/subsample_align.py b/python/dpd/subsample_align.py
index 20ae56b..20ae56b 100755
--- a/python/dpd/src/subsample_align.py
+++ b/python/dpd/subsample_align.py