Added new table of recent QSOs and more cleanup of the INI file plus a few others. (n1kdo#69)

* Added a chart that shows the last 10 worked contacts. This is in headless only.
   A future change is to set which charts are created in the config n1mm_view.ini file.
I also updated the init files to output StandardOutput and StandareError to journal as
    syslog is out of favor nowadays (I just learned that...)
Constants also have some mode changes to add potential bad modes if the source programhas an issue.
    Specifically, I made a mode of NoMode set to DATA as in some rare cases, TR4W could not have a sub-mode
    That usually happens on older radios without explicity sub-modes like PSK or USB-D.
I updated INSTALL_RASPI to mention the config file goes in just one place and also to
   suggest adding a symbolic link to /root/.config as I run dashboard with sudo
I updated rpi_install.sh to copy the ini file to the ./config directory.

* Fixed a typo in headless (was missing exception handler).
Removed config.sample file as the new file is now named n1mm_view.ini.sample

* Added the HEADLESS DWELL parameter

* Add a CREATE INDEX to index qso_log by timestamp for the last 10 qso table. This should be useful for the last_qso_logic too.

* Fixed a typo, an unused module and changed the qso loop. Notes int he original PR

* Reformatted the time column in the last 10 QSOs report. Also removed the Station column

Author: ny4i

INSTALL_RASPI.md

@@ -18,9 +18,9 @@ This last command will run for quite some time.
 The latest test took about 90 minutes with a good internet connection for a Raspberry Pi 3 B+ (18 minutes on a Pi 4B). We gave it a lot of things to do so you won't need to do them.
 While you are waiting, you could create a splash screen for your event: a 1000x1000 portable network graphics (.png) image in RGB format works great.   
 
-Copy the file config-sample.ini to ~/.config/n1mm_view.ini.
+Copy the file `n1mm_view.ini.sample` to `~/.config/n1mm_view.ini`. Note you can also put the file in your home directory (`~/`) or the script directory. But the file can only be in one of those places--not two or three.
 
-Note to pame it easier if you need to run dashboard.py as sudo, it is useful to create a symbolic link named /root/.config/n1mm_view.ini to point to /home/pi/.config/n1mm_view.ini.
+Note to make it easier if you need to run dashboard.py as sudo, it is useful to create a symbolic link named /root/.config/n1mm_view.ini to point to /home/pi/.config/n1mm_view.ini.
 Use the following command if so desired: 
  ```
 sudo ln -s  /home/pi/.config/n1mm_view.ini    /root/.config/n1mm_view.ini

config-sample.ini

@@ -116,6 +116,7 @@ def __init__(self, *args, **kw):
         self.DISPLAY_DWELL_TIME = cfg.getint('GLOBAL','DISPLAY_DWELL_TIME',fallback=6)
         self.DATA_DWELL_TIME = cfg.getint('GLOBAL','DATA_DWELL_TIME',fallback=60)
         self.HEADLESS_DWELL_TIME = cfg.getint('GLOBAL','HEADLESS_DWELL_TIME',fallback=180)
+        self.SKIP_TIMESTAMP_CHECK = cfg.getboolean('DEBUG','SKIP_TIMESTAMP_CHECK',fallback=False)
 
 
 

config.py

@@ -38,18 +38,18 @@ class Modes:
     """
     all the modes that are supported.
     """
-    MODES_LIST = ['N/A', 'CW', 'AM', 'FM', 'LSB', 'USB', 'RTTY', 'PSK', 'PSK31', 'PSK63', 'FT8', 'FT4']
+    MODES_LIST = ['N/A', 'CW', 'AM', 'FM', 'LSB', 'USB', 'SSB', 'RTTY', 'PSK', 'PSK31', 'PSK63', 'FT8', 'FT4', 'MFSK', 'NoMode', 'None']
     MODES = {elem: index for index, elem in enumerate(MODES_LIST)}
 
     """
     simplified modes for score reporting: CW, PHONE, DATA
     """
     SIMPLE_MODES_LIST = ['N/A', 'CW', 'PHONE', 'DATA']
-    MODE_TO_SIMPLE_MODE = [0, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3]
+    MODE_TO_SIMPLE_MODE = [0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2]
     SIMPLE_MODE_POINTS = [0, 2, 1, 2]  # n/a, CW, phone, digital
     SIMPLE_MODES = {'N/A': 0, 'CW': 1,
-                    'AM': 2, 'FM': 2, 'LSB': 2, 'USB': 2,
-                    'RTTY': 3, 'PSK': 3, 'PSK31': 3, 'PSK63': 3, 'FT8': 3, 'FT4': 3, 'MFSK': 3,
+                    'AM': 2, 'FM': 2, 'LSB': 2, 'USB': 2, 'SSB': 2, 'None': 2,
+                    'RTTY': 3, 'PSK': 3, 'PSK31': 3, 'PSK63': 3, 'FT8': 3, 'FT4': 3, 'MFSK': 3, 'NoMode': 3,
                     }
 
     @classmethod

constants.py

@@ -54,6 +54,7 @@ def create_tables(db, cursor):
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_station_id ON qso_log(station_id);')
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_section ON qso_log(section);')
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_qso_id ON qso_log(qso_id);')
+    cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_qso_timestamp ON qso_log(timestamp);')
     db.commit()
 
 
@@ -211,7 +212,8 @@ def get_last_qso(cursor) :
         message = 'Last QSO: %s %s %s on %s by %s at %s' % (
             row[1], row[2], row[3], constants.Bands.BANDS_TITLE[row[5]], row[4],
             datetime.utcfromtimestamp(row[0]).strftime('%H:%M:%S'))
-    logging.debug('%s' % (message))
+        logging.debug('%s' % (message))
+
     return last_qso_time, message
 
 
@@ -308,3 +310,34 @@ def get_qsos_by_section(cursor):
         qsos_by_section[row[0]] = row[1]
         logging.debug(f'Section {row[0]} {row[1]}')
     return qsos_by_section
+    
+def get_last_N_qsos(cursor, nQSOCount):
+    logging.info('get_last_N_qsos for last %d QSOs' % (nQSOCount))
+    qsos = []
+    cursor.execute('SELECT qso_id, timestamp, callsign, band_id, mode_id, operator.name, rx_freq, tx_freq, exchange, section, station.name \n'
+                   'FROM qso_log '
+                   'JOIN operator ON operator.id = operator_id\n'
+                   'JOIN station ON station.id = station_id\n'
+                   'ORDER BY timestamp DESC LIMIT %d;' % (nQSOCount))
+    for row in cursor:
+        qsos.append(( row[1] # raw timestamp 0
+                    ,row[2] # call 1
+                    ,constants.Bands.BANDS_TITLE[row[3]] # band 2
+                    ,constants.Modes.SIMPLE_MODES_LIST[constants.Modes.MODE_TO_SIMPLE_MODE[row[4]]]  # mode 3
+                    ,row[5] # operator callsign 4
+                    ,row[8] # exchange 5
+                    ,row[9] # section 6
+                    ,row[10] # station name 7
+                   ))
+        message = 'QSO: time=%sZ call=%s exchange=%s %s mode=%s band=%s operator=%s station=%s' % (
+             datetime.utcfromtimestamp(row[1]).strftime('%Y %b %d %H:%M:%S')
+            ,row[2] # callsign
+            ,row[8] # exchange
+            ,row[9] # section
+            ,constants.Modes.SIMPLE_MODES_LIST[constants.Modes.MODE_TO_SIMPLE_MODE[row[4]]]
+            ,constants.Bands.BANDS_TITLE[row[3]]
+            ,row[5] #operator
+            ,row[10] #station
+            )
+        logging.info('%s' % (message))
+    return qsos

dataaccess.py

@@ -210,7 +210,32 @@ def qso_classes_graph(size, qso_classes):
         values.append(d[0])
     return make_pie(size, values, labels, "QSOs by Class")
 
+def qso_table(size, qsos):
+    """
+    create the a table of the qso log
+    """
+    if len(qsos) == 0:
+        return None, (0, 0)
+
+    count = 0
+    cells = [['Time', 'Call', 'Band', 'Mode', 'Operator', 'Section']] #, 'Station']]
+    
+    for d in qsos[:10]:
+        cells.append( ['%s' % datetime.datetime.utcfromtimestamp(d[0]).strftime('%m-%d-%y %Tz') # Time
+                     ,'%s' % d[1] # Call
+                     ,'%s' % d[2] # Band
+                     ,'%s' % d[3] # Mode
+                     ,'%s' % d[4] # Operator
+                     ,'%s' % d[6] # Section
+       #              ,'%s' % d[7] # Station
+                     ])
+        count += 1
 
+    if count == 0:
+        return None, (0, 0)
+    else:
+        return draw_table(size, cells, "Last 10 QSOs")
+        
 def qso_operators_table(size, qso_operators):
     """
     create the Top 5 QSOs by Operators table

graphics.py

@@ -50,6 +50,7 @@ def create_images(size, image_dir, last_qso_timestamp):
     qsos_per_hour = []
     qsos_by_section = {}
     qso_classes = []
+    qsos = []
 
     db = None
     data_updated = False
@@ -70,7 +71,9 @@ def create_images(size, image_dir, last_qso_timestamp):
         last_qso_time, message = dataaccess.get_last_qso(cursor)
 
         logging.debug('old_timestamp = %s, timestamp = %s' % (last_qso_timestamp, last_qso_time))
-        if last_qso_time != last_qso_timestamp:
+        if config.SKIP_TIMESTAMP_CHECK: 
+           logging.warn('Skipping check for a recent QSO - Please just use this for debug - Review SKIP_TIMESTAMP_CHECK in ini file')
+        if last_qso_time != last_qso_timestamp or config.SKIP_TIMESTAMP_CHECK:
             # last_qso_time is passed as the result and updated in call to this function.
             logging.debug('data updated!')
             data_updated = True
@@ -95,8 +98,11 @@ def create_images(size, image_dir, last_qso_timestamp):
 
             # load QSOs by Section
             qsos_by_section = dataaccess.get_qsos_by_section(cursor)
+            
+            # load last 10 qsos
+            qsos = dataaccess.get_last_N_qsos(cursor, 10) # Note this returns last 10 qsos in reverse order so oldest is first
 
-        logging.debug('load data done')
+        logging.info('load data done')
     except sqlite3.OperationalError as error:
         logging.exception(error)
         return
@@ -115,78 +121,95 @@ def create_images(size, image_dir, last_qso_timestamp):
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_rates_table(size, operator_qso_rates)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_rates_table')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_operators_graph(size, qso_operators)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_operators_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_operators_table(size, qso_operators)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_operators_table')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_operators_table_all(size, qso_operators)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_operators_table_all')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)  
+        
         try:
             image_data, image_size = graphics.qso_stations_graph(size, qso_stations)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_stations_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_bands_graph(size, qso_band_modes)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_bands_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_modes_graph(size, qso_band_modes)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_modes_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_classes_graph(size, qso_classes)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_classes_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+        
         try:
             image_data, image_size = graphics.qso_rates_graph(size, qsos_per_hour)
             if image_data is not None:
                filename = makePNGTitle(image_dir, 'qso_rates_graph')
                graphics.save_image(image_data, image_size, filename)
         except Exception as e:
             logging.exception(e)
+            
+        try:
+            image_data, image_size = graphics.qso_table(size, qsos)
+            if image_data is not None:
+               filename = makePNGTitle(image_dir, 'last_qso_table')
+               graphics.save_image(image_data, image_size, filename)
+        except Exception as e:
+            logging.exception(e)
 
     # map gets updated every time so grey line moves
     try:
-        # There is a memory leak in the next code -- is there?
-        image_data, image_size = graphics.draw_map(size, qsos_by_section)
-        if image_data is not None:
-           filename = makePNGTitle(image_dir, 'sections_worked_map')
-           graphics.save_image(image_data, image_size, filename)
-           gc.collect()
+       # There is a memory leak in the next code -- is there?
+       image_data, image_size = graphics.draw_map(size, qsos_by_section)
+       if image_data is not None:
+          filename = makePNGTitle(image_dir, 'sections_worked_map')
+          graphics.save_image(image_data, image_size, filename)
+          gc.collect()
 
     except Exception as e:
         logging.exception(e)

headless.py

@@ -17,8 +17,8 @@ ExecStart=/home/pi/n1mm_view/collector.py
 Restart=always
 RestartSec=30
 StandardInput=tty
-StandardOutput=syslog
-StandardError=syslog
+StandardOutput=journal
+StandardError=journal
 TTYPath=/dev/tty2
 TTYReset=yes
 TTYVHangup=yes

init/n1mm_view_collector.service

@@ -6,18 +6,21 @@
 
 [Unit]
 Description=N1MM View Dashboard process
-After=gettty@tty3.service
+After=gettty@tty5.service
 
 [Service]
 User=pi
 Group=pi
 Type=simple
 WorkingDirectory=/home/pi/n1mm_view
+
+# Wait until we startup dashboard
+#ExecStartPre=/bin/sleep 30 
 ExecStart=/home/pi/n1mm_view/dashboard.py
 StandardInput=tty
-StandardOutput=syslog
-StandardError=syslog
-TTYPath=/dev/tty3
+StandardOutput=journal
+StandardError=journal
+TTYPath=/dev/tty5
 TTYReset=yes
 TTYVHangup=yes
 

init/n1mm_view_dashboard.service

@@ -17,8 +17,8 @@ ExecStart=/home/pi/n1mm_view/headless.py
 Restart=always
 RestartSec=30
 StandardInput=tty
-StandardOutput=syslog
-StandardError=syslog
+StandardOutput=journal
+StandardError=journal
 TTYPath=/dev/tty3
 TTYReset=yes
 TTYVHangup=yes

init/n1mm_view_headless.service

@@ -0,0 +1,53 @@
+# Change this file name to n1mm_view.ini and put it in ONLY one of the following three locations:
+#      ~/.config/n1mm_view.ini
+#      ~/n1mm_view.ini
+#      <script directory>/n1mm_view.ini
+# If you have the file in more than one of those locations, the script will abort with an error message.
+# This is to avoid multiple ini files making it harder to know where the settings are being read.
+
+# This is where you should set you local options for your site. At a minimum, change the following:
+# Under EVENT_INFO,
+#   set Name to something like <Your CLub Name> Field Day or WInter Field Day
+#   set the START_TIME and END_TIME
+#   set you QTH_ coordinates if you want to see a pin on the map
+# Optionally, set the LOGO_FILENAME to the proper logo
+#    For Winter Field Day, wget https://winterfieldday.org/img/wfda_logo.png
+#    For ARRL Field Day, grab the new logo each year
+#    If you are using the headless.py script to generate graphics to load to your website, 
+#    modify the POST_FILE_COMMAND to rsync the files or run a script if you want to do multiple steps.
+#    If you use a script like postCommands.sh, make it executable with chmod +x postCommands.sh
+
+
+
+[GLOBAL]
+DATABASE_FILENAME = n1mm_view.db
+DISPLAY_DWELL_TIME = 6
+DATA_DWELL_TIME = 60
+HEADLESS_DWELL_TIME = 120
+LOG_LEVEL = INFO
+LOGO_FILENAME = /home/pi/wfda_logo.png
+
+[EVENT INFO]
+NAME = Field Day
+START_TIME = 2025-01-25 15:00:00
+END_TIME = 2025-01-26 20:59:59
+QTH_LATITUDE = 27.9837941202094249
+QTH_LONGITUDE = -82.74670114956339
+
+[N1MM INFO]
+BROADCAST_PORT = 12060
+BROADCAST_ADDRESS = 192.168.1.255 
+LOG_FILE_NAME = FD2024-N4N.s3db
+
+[HEADLESS INFO]
+; Set IMAGE_DIR to None or the name of a directory on the system to write files. Note if using a Pi with an SD card only, use the ramdisk setup in the install process.
+; A sample value could be /mnt/ramdisk/n1mm_view/html
+IMAGE_DIR = None
+
+; The POST_FILE_COMMAND is used is to execute this command. You can use it to call rsync or a script.
+#POST_FILE_COMMAND = rsync -avz /mnt/ramdisk/n1mm_view/html/* user@sshserver:www/n1mm_view/html
+
+[FONT INFO]
+# If font seems too big, try 60 for VIEW_FONT and 100 for BIGGER_FONT
+VIEW_FONT = 64
+BIGGER_FONT = 180