Merge pull request #181 from projecthorus/testing

Updates to demodulators, experimental demod filtering improvements.

auto_rx/auto_rx.py

@@ -633,7 +633,8 @@ def main():
         _ozimux = OziUploader(
             ozimux_port = _ozi_port,
             payload_summary_port = _summary_port,
-            update_rate = config['ozi_update_rate'])
+            update_rate = config['ozi_update_rate'],
+            station=config['habitat_uploader_callsign'])
 
         exporter_objects.append(_ozimux)
         exporter_functions.append(_ozimux.add)

auto_rx/autorx/__init__.py

@@ -17,7 +17,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.1.0"
+__version__ = "1.1.1"
 
 
 # Global Variables
@@ -46,4 +46,4 @@
 
 
 # Scan result queue. 
-scan_results = Queue()
+scan_results = Queue()

auto_rx/autorx/aprs.py

@@ -153,7 +153,7 @@ def telemetry_to_aprs_position(sonde_data, object_name="<id>", aprs_comment="BOM
 
 
 
-def generate_station_object(callsign, lat, lon, comment="radiosonde_auto_rx SondeGate v<version>", icon='/r'):
+def generate_station_object(callsign, lat, lon, comment="radiosonde_auto_rx SondeGate v<version>", icon='/r', position_report=False):
     ''' Generate a station object '''
 
     # Pad or limit the station callsign to 9 characters, if it is to long or short.
@@ -192,7 +192,13 @@ def generate_station_object(callsign, lat, lon, comment="radiosonde_auto_rx Sond
     _aprs_comment = _aprs_comment.replace('<version>', auto_rx_version)
 
     # Generate output string
-    out_str = ";%s*%sh%s%s%s%s%s" % (callsign, _aprs_timestamp, lat_str, icon[0], lon_str, icon[1], _aprs_comment)
+    if position_report:
+        # Produce a position report with no timestamp, as per page 32 of http://www.aprs.org/doc/APRS101.PDF
+        out_str = "!%s%s%s%s%s" % (lat_str, icon[0], lon_str, icon[1], _aprs_comment)
+
+    else:
+        # Produce an object string
+        out_str = ";%s*%sh%s%s%s%s%s" % (callsign, _aprs_timestamp, lat_str, icon[0], lon_str, icon[1], _aprs_comment)
 
     return out_str
 
@@ -388,15 +394,16 @@ def beacon_station_position(self):
         ''' Send a station position beacon into APRS-IS '''
         if self.station_beacon['enabled']:
             # Generate the station position packet
-            # Note - this is generated as an APRS object.
+            # Note - this is now generated as an APRS position report, for radiosondy.info compatability.
             _packet = generate_station_object(self.aprs_callsign,
                 self.station_beacon['position'][0], 
                 self.station_beacon['position'][1],
                 self.station_beacon['comment'], 
-                self.station_beacon['icon'])
+                self.station_beacon['icon'],
+                position_report=True)
 
-            # Send the packet
-            self.aprsis_upload(self.aprs_callsign, _packet, igate=False)
+            # Send the packet as an iGated packet.
+            self.aprsis_upload(self.aprs_callsign, _packet, igate=True)
             self.last_user_position_upload = time.time()
 
 

auto_rx/autorx/decode.py

@@ -325,7 +325,7 @@ def generate_decoder_command(self):
                 decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
 
             # DFM decoder
-            decode_cmd += "./dfm09ecc -vv --ecc --json --dist --auto 2>/dev/null"
+            decode_cmd += "./dfm09mod -vv --ecc --json --dist --auto 2>/dev/null"
 
         elif self.sonde_type == "M10":
             # M10 Sondes
@@ -400,7 +400,7 @@ def generate_decoder_command_experimental(self):
             _upper = int(0.475 * _sdr_rate)
             _freq = int(self.sonde_freq - _sdr_rate*_offset)
 
-            decode_cmd = "%s %s-p %d -d %s %s-M raw -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
+            decode_cmd = "%s %s-p %d -d %s %s-M raw -F9 -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
             # Add in tee command to save IQ to disk if debugging is enabled.
             if self.save_decode_iq:
                 decode_cmd += " tee decode_IQ_%s.bin |" % str(self.device_idx)
@@ -449,7 +449,7 @@ def generate_decoder_command_experimental(self):
             _upper = int(0.475 * _sdr_rate)
             _freq = int(self.sonde_freq - _sdr_rate*_offset)
 
-            decode_cmd = "%s %s-p %d -d %s %s-M raw -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
+            decode_cmd = "%s %s-p %d -d %s %s-M raw -F9 -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
 
             # Add in tee command to save IQ to disk if debugging is enabled.
             if self.save_decode_iq:
@@ -466,9 +466,6 @@ def generate_decoder_command_experimental(self):
 
         elif self.sonde_type == "DFM":
             # DFM06/DFM09 Sondes.
-            # As of 2019-02-10, dfm09ecc auto-detects if the signal is inverted,
-            # so we don't need to specify an invert flag.
-            # 2019-02-27: Added the --dist flag, which should reduce bad positions a bit.
 
             _sdr_rate = 50000
             _baud_rate = 2500
@@ -477,21 +474,21 @@ def generate_decoder_command_experimental(self):
             _upper = int(0.475 * _sdr_rate)
             _freq = int(self.sonde_freq - _sdr_rate*_offset)
 
-            decode_cmd = "%s %s-p %d -d %s %s-M raw -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
+            decode_cmd = "%s %s-p %d -d %s %s-M raw -F9 -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
 
             # Add in tee command to save IQ to disk if debugging is enabled.
             if self.save_decode_iq:
                 decode_cmd += " tee decode_IQ_%s.bin |" % str(self.device_idx)
 
             decode_cmd += "./fsk_demod --cs16 -b %d -u %d %s2 %d %d - - %s |" % (_lower, _upper, _stats_command_1, _sdr_rate, _baud_rate, _stats_command_2)
-            decode_cmd += " python ./test/bit_to_samples.py %d %d | sox -t raw -r %d -e unsigned-integer -b 8 -c 1 - -r %d -b 8 -t wav - 2>/dev/null |" % (_sdr_rate, _baud_rate, _sdr_rate, _sdr_rate)
+            #decode_cmd += " python ./test/bit_to_samples.py %d %d | sox -t raw -r %d -e unsigned-integer -b 8 -c 1 - -r %d -b 8 -t wav - 2>/dev/null |" % (_sdr_rate, _baud_rate, _sdr_rate, _sdr_rate)
 
             # Add in tee command to save audio to disk if debugging is enabled.
             if self.save_decode_audio:
                 decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
 
             # DFM decoder
-            decode_cmd += "./dfm09ecc -vv --ecc --json --dist --auto 2>/dev/null"
+            decode_cmd += "./dfm09mod -vv --ecc --json --dist --auto --bin 2>/dev/null"
 
         elif self.sonde_type == "M10":
             # M10 Sondes
@@ -504,7 +501,7 @@ def generate_decoder_command_experimental(self):
             _upper = int(0.475 * _sdr_rate)
             _freq = int(self.sonde_freq - _sdr_rate*_offset)
 
-            decode_cmd = "%s %s-p %d -d %s %s-M raw -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
+            decode_cmd = "%s %s-p %d -d %s %s-M raw -F9 -s %d -f %d 2>/dev/null |" % (self.sdr_fm, bias_option, int(self.ppm), str(self.device_idx), gain_param, _sdr_rate, _freq)
 
             # Add in tee command to save IQ to disk if debugging is enabled.
             if self.save_decode_iq:
@@ -605,25 +602,25 @@ def decoder_thread(self):
 
 
     def handle_decoder_line(self, data):
-        """ Handle a line of output from the decoder subprocess.
+        """ Handle a line of output from the decoder subprocess, and pass it onto all of the telemetry
+            exporters.
 
         Args:
             data (str, bytearray): One line of text output from the decoder subprocess.
 
         Returns:
             bool:   True if the line was decoded to a JSON object correctly, False otherwise.
         """
-
-        # Don't even try and decode lines which don't start with a '{'
-        # These may be other output from the decoder, which we shouldn't try to parse.
 
         # Catch 'bad' first characters.
         try:
             _first_char = data.decode('ascii')[0]
         except UnicodeDecodeError:
             return
 
-        # Catch non-JSON object lines.
+        # Don't even try and decode lines which don't start with a '{'
+        # These may be other output from the decoder, which we shouldn't try to parse.
+        # TODO: Perhaps we should add the option to log the raw data output from the decoders? 
         if data.decode('ascii')[0] != '{':
             return
 
@@ -651,12 +648,15 @@ def handle_decoder_line(self, data):
                     _telemetry[_field] = self.DECODER_OPTIONAL_FIELDS[_field]
 
 
-            # Check for an encrypted flag (this indicates a sonde that we cannot decode telemetry from.)
+            # Check for an encrypted flag, and check if it is set.
+            # Currently encrypted == true indicates an encrypted RS41-SGM. There's no point
+            # trying to decode this, so we close the decoder at this point.
             if 'encrypted' in _telemetry:
-                self.log_error("Radiosonde %s has encrypted telemetry (possible RS41-SGM)! We cannot decode this, closing decoder." % _telemetry['id'])
-                self.exit_state = "Encrypted"
-                self.decoder_running = False
-                return False
+                if _telemetry['encrypted']:
+                    self.log_error("Radiosonde %s has encrypted telemetry (Possible encrypted RS41-SGM)! We cannot decode this, closing decoder." % _telemetry['id'])
+                    self.exit_state = "Encrypted"
+                    self.decoder_running = False
+                    return False
 
             # Check the datetime field is parseable.
             try:
@@ -665,8 +665,15 @@ def handle_decoder_line(self, data):
                 self.log_error("Invalid date/time in telemetry dict - %s (Sonde may not have GPS lock)" % str(e))
                 return False
 
-            # Add in the sonde frequency and type fields.
-            _telemetry['type'] = self.sonde_type
+            # Add in the sonde type field.
+            # If we are provided with a subtype field from the decoder, use this,
+            # otherwise use the detected sonde type.
+            if 'subtype' in _telemetry:
+                _telemetry['type'] = _telemetry['subtype']
+            else:
+                _telemetry['type'] = self.sonde_type
+
+            # TODO: Use frequency data provided by the decoder, if available.
             _telemetry['freq_float'] = self.sonde_freq/1e6
             _telemetry['freq'] = "%.3f MHz" % (self.sonde_freq/1e6)
 
@@ -684,11 +691,12 @@ def handle_decoder_line(self, data):
                 # Check we have GPS lock.
                 if _telemetry['sats'] < 4:
                     # No GPS lock means an invalid time, which means we can't accurately calculate a unique ID.
+                    # We need to quit at this point before the telemetry processing gos any further.
                     self.log_error("iMet sonde has no GPS lock - discarding frame.")
                     return False
 
                 # Fix up the time.
-                _telemetry['datetime_dt'] = imet_fix_datetime(_telemetry['datetime'])
+                _telemetry['datetime_dt'] = fix_datetime(_telemetry['datetime'])
                 # Generate a unique ID based on the power-on time and frequency, as iMet sondes don't send one.
                 # Latch this ID and re-use it for the entire decode run.
                 if self.imet_id == None:

auto_rx/autorx/ozimux.py

@@ -43,7 +43,8 @@ class OziUploader(object):
     def __init__(self,
         ozimux_port = None,
         payload_summary_port = None,
-        update_rate = 5
+        update_rate = 5,
+        station = "auto_rx"
         ):
         """ Initialise an OziUploader Object.
 
@@ -56,6 +57,7 @@ def __init__(self,
         self.ozimux_port = ozimux_port
         self.payload_summary_port = payload_summary_port
         self.update_rate = update_rate
+        self.station = station
 
         # Input Queue. 
         self.input_queue = Queue()
@@ -131,6 +133,7 @@ def send_payload_summary(self, telemetry):
 
             packet = {
                 'type' : 'PAYLOAD_SUMMARY',
+                'station': self.station,
                 'callsign' : telemetry['id'],
                 'latitude' : telemetry['lat'],
                 'longitude' : telemetry['lon'],
@@ -142,7 +145,7 @@ def send_payload_summary(self, telemetry):
                 # Additional fields specifically for radiosondes
                 'model': telemetry['type'],
                 'freq': telemetry['freq'],
-                'temp': telemetry['temp']
+                'temp': telemetry['temp'],
             }
 
             # Add in any extra fields we may care about.

auto_rx/autorx/sonde_specific.py

@@ -9,13 +9,11 @@
 import hashlib
 from dateutil.parser import parse
 
-#
-#	iMet Radiosonde Functions
-#
 
-def imet_fix_datetime(datetime_str, local_dt_str = None):
+
+def fix_datetime(datetime_str, local_dt_str = None):
 	'''
-	Given a HH:MM:SS string from an iMet telemetry sentence, produce a complete timestamp, using the current system time as a guide for the date.
+	Given a HH:MM:SS string from a telemetry sentence, produce a complete timestamp, using the current system time as a guide for the date.
 	'''
 
 	if local_dt_str is None:
@@ -50,6 +48,10 @@ def imet_fix_datetime(datetime_str, local_dt_str = None):
 		return _imet_dt
 
 
+#
+#	iMet Radiosonde Functions
+#
+
 def imet_unique_id(telemetry, custom=""):
 	'''
 	Generate a 'unique' imet radiosonde ID based on the power-on time, frequency, and an optional location code.

auto_rx/autorx/utils.py

@@ -30,7 +30,7 @@
 
 
 # List of binaries we check for on startup
-REQUIRED_RS_UTILS = ['dft_detect', 'dfm09ecc', 'm10', 'imet1rs_dft', 'rs41mod', 'rs92mod', 'fsk_demod']
+REQUIRED_RS_UTILS = ['dft_detect', 'dfm09mod', 'm10', 'imet1rs_dft', 'rs41mod', 'rs92mod', 'fsk_demod']
 
 def check_rs_utils():
     """ Check the required RS decoder binaries exist

auto_rx/build.sh

@@ -21,8 +21,7 @@ cd ../demod/mod/
 gcc -c demod_mod.c -w
 gcc -c bch_ecc_mod.c -w
 gcc rs41mod.c demod_mod.o bch_ecc_mod.o -lm -o rs41mod -w
-# Holding off on using the new DFM decoder until the DFM17/15 ID issue is resolved.
-#gcc dfm09mod.c demod_mod.o -lm -o dfm09mod -w
+gcc dfm09mod.c demod_mod.o -lm -o dfm09mod -w
 gcc rs92mod.c demod_mod.o bch_ecc_mod.o -lm -o rs92mod -w
 #gcc lms6mod.c demod_mod.o bch_ecc_mod.o -lm -o lms6mod -w
 #gcc m10mod.c demod_mod.o -lm -o m10mod -w
@@ -57,7 +56,8 @@ cp ../utils/fsk_demod .
 cp ../imet/imet1rs_dft .
 
 cp ../demod/mod/rs41mod .
-#cp ../demod/mod/dfm09mod .
+cp ../demod/mod/dfm09mod .
+#cp ../demod/mod/m10mod .
 cp ../demod/mod/rs92mod .
 #cp ../demod/mod/lms6mod .
 

auto_rx/station.cfg.example

@@ -90,6 +90,7 @@ station_alt = 0.0
 # If you know the WMO number of the launch site, then this would be a good value to use. 
 # Otherwise the ICAO Code of the airport nearest to the launch site would also work.
 # If you are not expecting to RX iMet sondes, then this can be left at its default.
+# NOTE: Only change this if you are 100% sure you will only be receiving sondes from a single launch location!
 
 station_code = SONDE
 
@@ -191,7 +192,8 @@ station_beacon_commment = radiosonde_auto_rx SondeGate v<version>
 # Note that the two characters that define the icon need to be concatenated. Examples:
 # Antenna Tower = /r
 # House with Yagi = /y
-station_beacon_icon = /r
+# Satellite Dish = /`    (This is required if you want to show up on radiosondy.info's station list.)
+station_beacon_icon = /`
 
 
 
@@ -200,14 +202,13 @@ station_beacon_icon = /r
 ###########################
 # Settings for pushing data into Chasemapper and/or OziPlotter
 # Oziplotter receives data via a basic CSV format, via UDP.
-# Chasemapper can receive data in either the basic CSV format, or in a more descriptive JSON format.
-# WARNING - This should not be enabled in a multi-SDR configuration, as OziExplorer currently has no way of differentiating
-# between sonde IDs.
+# Chasemapper can receive data in either the basic CSV format, or in the more descriptive 'payload summary' JSON format.
 [oziplotter]
 # How often to output data (seconds)
 ozi_update_rate = 5
 
 # Enable the 'OziMux' basic CSV output
+# Note - this cannot be enabled in a multi-SDR configuration.
 ozi_enabled = False
 
 # OziMux UDP Broadcast output port.