Merge pull request #727 from DJ2LS/develop

improving packet collision detection and server shutdown

freedata_gui/package.json

@@ -2,7 +2,7 @@
   "name": "FreeDATA",
   "description": "FreeDATA Client application for connecting to FreeDATA server",
   "private": true,
-  "version": "0.15.8-alpha",
+  "version": "0.15.9-alpha",
   "main": "dist-electron/main/index.js",
   "scripts": {
     "start": "vite",
@@ -33,7 +33,7 @@
   },
   "homepage": "https://freedata.app",
   "dependencies": {
-    "@electron/notarize": "2.2.1",
+    "@electron/notarize": "2.3.2",
     "@electron/universal": "2.0.1",
     "@popperjs/core": "2.11.8",
     "@vueuse/electron": "10.7.2",
@@ -41,10 +41,10 @@
     "bootstrap": "5.3.2",
     "bootstrap-icons": "1.11.3",
     "browser-image-compression": "2.0.2",
-    "chart.js": "4.4.2",
+    "chart.js": "4.4.3",
     "chartjs-plugin-annotation": "3.0.1",
     "electron-log": "5.1.2",
-    "emoji-picker-element": "1.21.0",
+    "emoji-picker-element": "1.21.3",
     "emoji-picker-element-data": "1.6.0",
     "file-saver": "2.0.5",
     "gridstack": "10.0.1",
@@ -73,12 +73,12 @@
     "eslint-plugin-prettier": "5.1.3",
     "eslint-plugin-promise": "6.1.1",
     "eslint-plugin-vue": "9.22.0",
-    "typescript": "5.3.3",
+    "typescript": "5.4.5",
     "vite": "5.1.7",
     "vite-plugin-electron": "0.28.2",
     "vite-plugin-electron-renderer": "0.14.5",
     "vitest": "1.4.0",
     "vue": "3.4.21",
-    "vue-tsc": "1.8.27"
+    "vue-tsc": "2.0.19"
   }
 }

freedata_gui/src/js/eventHandler.js

@@ -57,6 +57,7 @@ export function connectionFailed(endpoint, event) {
 }
 export function stateDispatcher(data) {
   data = JSON.parse(data);
+
   //Leave commented when not needed, otherwise can lead to heap overflows due to the amount of data logged
   //console.log(data);
   if (data["type"] == "state-change" || data["type"] == "state") {
@@ -69,7 +70,9 @@ export function stateDispatcher(data) {
     stateStore.dbfs_level_percent = Math.round(
       Math.pow(10, data["audio_dbfs"] / 20) * 100,
     );
-
+    //Accept radio status from here as well, saves us from having to wait for an update from the radio manager
+    //Fixes the health rigctl being offline on startup
+    stateStore.radio_status = data["radio_status"];
     stateStore.channel_busy_slot = data["channel_busy_slot"];
     stateStore.beacon_state = data["is_beacon_running"];
     stateStore.away_from_key = data["is_away_from_key"];
@@ -80,8 +83,6 @@ export function stateDispatcher(data) {
   }
 
   if (data["type"] == "radio-change" || data["type"] == "radio") {
-    console.log(data);
-
     stateStore.s_meter_strength_raw = Math.round(data["s_meter_strength"]);
     stateStore.s_meter_strength_percent = Math.round(
       Math.pow(10, data["s_meter_strength"] / 20) * 100,

freedata_server/audio.py

@@ -38,7 +38,7 @@ def get_audio_devices():
             target=fetch_audio_devices, args=(proxy_input_devices, proxy_output_devices)
         )
         proc.start()
-        proc.join()
+        proc.join(3)
 
         # additional logging for audio devices
         # log.debug("[AUD] get_audio_devices: input_devices:", list=f"{proxy_input_devices}")
@@ -358,5 +358,6 @@ def calculate_fft(data, fft_queue, states) -> None:
         print(f"[MDM] calculate_fft: Exception: {err}")
 
 def terminate():
+    print("terminating audio instance...")
     if sd._initialized:
         sd._terminate()

freedata_server/command_arq_raw.py

@@ -5,7 +5,8 @@ import base64
 from queue import Queue
 from arq_session_iss import ARQSessionISS
 from arq_data_type_handler import ARQ_SESSION_TYPES
-
+import numpy as np
+import threading
 
 class ARQRawCommand(TxCommand):
 
@@ -26,6 +27,12 @@ class ARQRawCommand(TxCommand):
             self.emit_event(event_queue)
             self.logger.info(self.log_message())
 
+            # wait some random time and wait if we have an ongoing codec2 transmission
+            # on our channel. This should prevent some packet collision
+            random_delay = np.random.randint(0, 6)
+            threading.Event().wait(random_delay)
+            self.state_manager.channel_busy_condition_codec2.wait(5)
+
             prepared_data, type_byte = self.arq_data_type_handler.prepare(self.data, self.type)
 
             iss = ARQSessionISS(self.config, modem, self.dxcall, self.state_manager, prepared_data, type_byte)

freedata_server/command_message_send.py

@@ -7,6 +7,9 @@ from message_p2p import MessageP2P
 from arq_data_type_handler import ARQ_SESSION_TYPES
 from message_system_db_manager import DatabaseManager
 from message_system_db_messages import DatabaseManagerMessages
+import threading
+import numpy as np
+
 
 class SendMessageCommand(TxCommand):
     """Command to send a P2P message using an ARQ transfer session
@@ -38,6 +41,13 @@ class SendMessageCommand(TxCommand):
             message_dict = DatabaseManagerMessages(self.event_manager).get_message_by_id(first_queued_message["id"])
             message = MessageP2P.from_api_params(message_dict['origin'], message_dict)
 
+            # wait some random time and wait if we have an ongoing codec2 transmission
+            # on our channel. This should prevent some packet collision
+            random_delay = np.random.randint(0, 6)
+            threading.Event().wait(random_delay)
+            while self.state_manager.is_receiving_codec2_signal():
+                threading.Event().wait(0.1)
+
             # Convert JSON string to bytes (using UTF-8 encoding)
             payload = message.to_payload().encode('utf-8')
             json_bytearray = bytearray(payload)

freedata_server/config.ini.example

@@ -7,8 +7,8 @@ mycall = AA1AAA
 mygrid = JN48ea
 myssid = 1
 ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-enable_explorer = True
-enable_stats = True
+enable_explorer = False
+enable_stats = False
 
 [AUDIO]
 input_device = 5a1c
@@ -48,7 +48,7 @@ enable_hmac = False
 enable_morse_identifier = False
 respond_to_cq = True
 tx_delay = 50
-maximum_bandwidth = 2375
+maximum_bandwidth = 2438
 enable_socket_interface = False
 
 [SOCKET_INTERFACE]

freedata_server/demodulator.py

@@ -362,6 +362,7 @@ class Demodulator():
                     self.MODE_DICT[mode]["decode"] = decode
 
     def shutdown(self):
+        print("shutting down demodulators...")
         self.shutdown_flag.set()
         for mode in self.MODE_DICT:
-            self.MODE_DICT[mode]['decoding_thread'].join()
+            self.MODE_DICT[mode]['decoding_thread'].join(3)

freedata_server/frame_handler.py

@@ -158,6 +158,9 @@ class FrameHandler():
             distance = maidenhead.distance_between_locators(self.config['STATION']['mygrid'], self.details['frame']['gridsquare'])
             event['distance_kilometers'] = distance['kilometers']
             event['distance_miles'] = distance['miles']
+            
+        if "flag" in self.details['frame'] and "AWAY_FROM_KEY" in self.details['frame']["flag"]:
+            event['away_from_key'] = self.details['frame']["flag"]["AWAY_FROM_KEY"]
 
         return event
 

freedata_server/frame_handler_cq.py

@@ -1,9 +1,11 @@
+import threading
+
 import frame_handler_ping
 import helpers
 import data_frame_factory
 import frame_handler
 from message_system_db_messages import DatabaseManagerMessages
-
+import numpy as np
 
 class CQFrameHandler(frame_handler.FrameHandler):
 
@@ -25,9 +27,14 @@ class CQFrameHandler(frame_handler.FrameHandler):
 
     def send_ack(self):
         factory = data_frame_factory.DataFrameFactory(self.config)
-        qrv_frame = factory.build_qrv(
-            self.details['snr']
-        )
+        qrv_frame = factory.build_qrv(self.details['snr'])
+
+        # wait some random time and wait if we have an ongoing codec2 transmission
+        # on our channel. This should prevent some packet collision
+        random_delay = np.random.randint(0, 6)
+        threading.Event().wait(random_delay)
+        self.states.channel_busy_condition_codec2.wait(5)
+
         self.transmit(qrv_frame)
 
         if self.config["MESSAGES"]["enable_auto_repeat"]:

freedata_server/radio_manager.py

@@ -67,9 +67,9 @@ class RadioManager:
                 self.state_manager.set_radio("radio_swr", parameters['swr'])
 
             self.state_manager.set_radio("s_meter_strength", parameters['strength'])
-            time.sleep(self.refresh_rate)
+            threading.Event().wait(self.refresh_rate)
 
     def stop(self):
-        self.radio.disconnect()
         self.stop_event.set()
+        self.radio.disconnect()
         self.radio.stop_service()

freedata_server/rigctld.py

@@ -19,6 +19,7 @@ class radio:
 
         self.connection = None
         self.connected = False
+        self.shutdown = False
         self.await_response = threading.Event()
         self.await_response.set()
 
@@ -42,7 +43,10 @@ class radio:
         self.connect()
 
     def connect(self):
-
+        print(self.hostname)
+        print(self.port)
+        if self.shutdown:
+            return
         try:
             self.connection = socket.create_connection((self.hostname, self.port), timeout=self.timeout)
             self.connected = True
@@ -54,6 +58,7 @@ class radio:
             self.states.set_radio("radio_status", False)
 
     def disconnect(self):
+        self.shutdown = True
         self.connected = False
         if self.connection:
             self.connection.close()
@@ -347,6 +352,9 @@ class radio:
                     self.log.info(f"Attempting to start rigctld using binary found at: {binary_path}")
                     self.rigctld_process = helpers.kill_and_execute(binary_path, additional_args)
                     self.log.info("Successfully executed rigctld.")
+                    print(self.rigctld_process)
+                    print(additional_args)
+                    print(binary_paths)
                     break  # Exit the loop after successful execution
                 except Exception as e:
                     pass

freedata_server/schedule_manager.py

@@ -19,7 +19,7 @@ class ScheduleManager:
         self.event_manager = event_manager
         self.config = self.config_manager.read()
 
-        self.scheduler = sched.scheduler(time.time, time.sleep)
+        self.scheduler = sched.scheduler(time.time, threading.Event().wait)
         self.events = {
             'check_for_queued_messages': {'function': self.check_for_queued_messages, 'interval': 5},
             'explorer_publishing': {'function': self.push_to_explorer, 'interval': 60},
@@ -41,7 +41,7 @@ class ScheduleManager:
         """Start scheduling events and run the scheduler in a separate thread."""
 
         # wait some time for the modem to be ready
-        threading.Event().wait(timeout=0.5)
+        threading.Event().wait(timeout=0.1)
 
         # get actual freedata_server instance
         self.modem = modem
@@ -65,7 +65,7 @@ class ScheduleManager:
 
         # Wait for the scheduler thread to finish
         if self.scheduler_thread:
-            self.scheduler_thread.join()
+            self.scheduler_thread.join(3)
         print("done")
     def transmit_beacon(self):
         try:
@@ -100,8 +100,5 @@ class ScheduleManager:
             except Exception as e:
                 print(e)
 
-        # let's wait some random time for decreasing chance of packet colission.
-        threading.Event().wait(np.random.randint(3,10))
-
         return
 

freedata_server/server.py

@@ -38,7 +38,7 @@ from schedule_manager import ScheduleManager
 app = Flask(__name__)
 CORS(app, resources={r"/*": {"origins": "*"}})
 sock = Sock(app)
-MODEM_VERSION = "0.15.8-alpha"
+MODEM_VERSION = "0.15.9-alpha"
 
 # set config file to use
 def set_config():
@@ -350,19 +350,18 @@ def signal_handler(sig, frame):
     print("Received SIGINT......")
     stop_server()
 
+
 def stop_server():
-    if hasattr(app, 'schedule_manager'):
-        app.schedule_manager.stop()
 
     if hasattr(app, 'radio_manager'):
         app.radio_manager.stop()
-
-
+    import time
+    start_time = time.time()
+    if hasattr(app, 'schedule_manager'):
+        app.schedule_manager.stop()
+    print(time.time()-start_time)
     if hasattr(app, 'service_manager'):
 
-        if hasattr(app, 'socket_interface_manager') and app.socket_interface_manager:
-            app.socket_interface_manager.stop_servers()
-
         if hasattr(app.service_manager, 'modem_service') and app.service_manager.modem_service:
             app.service_manager.shutdown()
 
@@ -370,12 +369,17 @@ def stop_server():
             #    app.service_manager.modem.sd_input_stream.stop
             app.service_manager.modem.demodulator.shutdown()
 
-        audio.terminate()
+    if hasattr(app, 'socket_interface_manager') and app.socket_interface_manager:
+        app.socket_interface_manager.stop_servers()
 
+    audio.terminate()
 
     print("Shutdown completed")
     print(".........................")
-    sys.exit(0)
+    try:
+        sys.exit(0)
+    except Exception as e:
+        print(e)
 
 def main():
     # Register the signal handler for SIGINT

freedata_server/service_manager.py

@@ -145,6 +145,7 @@ class SM:
         del self.app.radio_manager
 
     def shutdown(self):
+        print("shutting down service manager...")
         self.modem_service.put("stop")
         self.shutdown_flag.set()
-        self.runner_thread.join()
+        self.runner_thread.join(3)

freedata_server/socket_interface.py

@@ -185,12 +185,12 @@ class SocketInterfaceHandler:
         # Gracefully shutdown the server
         if self.command_server:
             self.command_server.shutdown()
-            self.command_server_thread.join()
+            self.command_server_thread.join(3)
 
             del self.command_server
         if self.data_server:
             self.data_server.shutdown()
-            self.data_server_thread.join()
+            self.data_server_thread.join(3)
             del self.data_server
 
-        self.log(f"Interfaces stopped")
+        self.log(f"socket interfaces stopped")

tests/test_arq_session.py

@@ -65,6 +65,8 @@ class TestARQSession(unittest.TestCase):
 
         # ISS
         cls.iss_state_manager = StateManager(queue.Queue())
+        cls.iss_state_manager.set_channel_busy_condition_codec2(False)
+
         cls.iss_event_manager = EventManager([queue.Queue()])
         cls.iss_event_queue = queue.Queue()
         cls.iss_state_queue = queue.Queue()
@@ -76,6 +78,8 @@ class TestARQSession(unittest.TestCase):
 
         # IRS
         cls.irs_state_manager = StateManager(queue.Queue())
+        cls.iss_state_manager.set_channel_busy_condition_codec2(False)
+
         cls.irs_event_manager = EventManager([queue.Queue()])
         cls.irs_event_queue = queue.Queue()
         cls.irs_state_queue = queue.Queue()

tests/test_message_protocol.py

@@ -66,6 +66,8 @@ class TestMessageProtocol(unittest.TestCase):
 
         # ISS
         cls.iss_state_manager = StateManager(queue.Queue())
+        cls.iss_state_manager.set_channel_busy_condition_codec2(False)
+
         cls.iss_event_manager = EventManager([queue.Queue()])
         cls.iss_event_queue = queue.Queue()
         cls.iss_state_queue = queue.Queue()

tests/test_protocols.py

@@ -64,8 +64,10 @@ class TestProtocols(unittest.TestCase):
         self.shortcutTransmission(event_frame)
         self.assertEventReceivedType('PING_ACK')
         print("PING/PING ACK CHECK SUCCESSFULLY")
+
     def testCQWithQRV(self):
         self.config['MODEM']['respond_to_cq'] = True
+        self.state_manager.set_channel_busy_condition_codec2(False)
 
         api_params = {}
         cmd = CQCommand(self.config, self.state_manager, self.event_manager, api_params)

tools/Linux/install-freedata-linux.sh

@@ -19,9 +19,13 @@
 #
 #
 # Changelog:
+# 1.6:	22 May 2024
+#	Reflect directory name changes in prep for merging develop to main
+#
 # 1.5:	12 May 2024
 #	"dr-freedata-001" branch of codec2 merged to main so we don't
 #	need to checkout that branch specifically
+#
 # 1.4:	05 May 2024
 #	Change to "dr-freedata-001" branch of codec2 for develop mode
 #	Added comments in scripts and README.txt for config file location
@@ -267,12 +271,7 @@ pip install --upgrade -r requirements.txt
 echo "*************************************************************************"
 echo "Changing into the server directory"
 echo "*************************************************************************"
-if [ "$args" == "develop" ];
-then
-	cd freedata_server/lib
-else
-	cd modem/lib
-fi
+cd freedata_server/lib
 
 echo "*************************************************************************"
 echo "Checking and removing any old codec2 libraries"
@@ -298,12 +297,6 @@ else
 	exit 1
 fi
 	
-# Not currently needed
-# if [ "$args" == "develop" ];
-# then
-# 	git checkout dr-freedata-001
-# fi
-
 echo "*************************************************************************"
 echo "Setting up the codec2 build"
 echo "*************************************************************************"
@@ -326,12 +319,7 @@ echo "*************************************************************************"
 echo "Building the FreeDATA GUI frontend"
 echo "*************************************************************************"
 cd ../../../..
-if [ "$args" == "develop" ];
-then
-	cd freedata_gui
-else
-	cd gui
-fi
+cd freedata_gui
 npm i
 npm audit fix --force
 npm i

tools/Linux/run-freedata-linux.sh

@@ -15,6 +15,8 @@
 # We expect the config.ini file to be at $HOME/.config/FreeDATA/config.ini
 # If it isn't found, we copy config.ini.example there
 #
+# 1.7:  22 May 2024
+#	Slightly change the way we shutdown the server
 # 1.6:  05 May 2024
 #	Don't stop rigctld if it was started separate from FreeDATA
 #	We only want to clean up FreeDATA initiated processes
@@ -128,13 +130,16 @@ npm start > ../../FreeDATA-client.log 2>&1
 echo "*************************************************************************"
 echo "Stopping the server component"
 echo "*************************************************************************"
-kill $serverpid
+kill -INT $serverpid
+
+# Give time for the server to clean up rigctld if needed
+sleep 5s
 
 # If rigctld was already running before starting FreeDATA, leave it alone
 # otherwise we should clean it up
 if [ -z "$checkrigexist" ];
 then
-	# rigctld was started by FreeDATA and should have stoppped when the 
+	# rigctld was started by FreeDATA and should have stopped when the 
 	# server exited.  If it didn't, stop it now.
 	checkrigctld=`ps auxw | grep -i rigctld | grep -v grep`
 	if [ ! -z "$checkrigctld" ];

tools/Windows/GUI-Install-Requirements.bat

@@ -1,8 +1,8 @@
 @echo off
-REM Place this batch file in FreeData/gui and then run it
-REM ie. c:\FD-Src\gui
+REM Place this batch file in FreeData/freedata_gui and then run it
+REM ie. c:\FD-Src\freedata_gui
 echo Install requirements for GUI...
 
 call npm install
 
-pause
+pause

tools/Windows/GUI-Launch.bat

@@ -1,5 +1,5 @@
-REM Place this batch file in FreeData/gui and then run it
-REM ie. c:\FD-Src\gui
+REM Place this batch file in FreeData/freedata_gui and then run it
+REM ie. c:\FD-Src\freedata_gui
 
 call npm start
-pause
+pause

tools/Windows/GUI-Update-Requirements.bat

@@ -1,8 +1,8 @@
 @echo off
-REM Place this batch file in FreeData/gui and then run it
-REM ie. c:\FD-Src\gui
+REM Place this batch file in FreeData/freedata_gui and then run it
+REM ie. c:\FD-Src\freedata_gui
 echo Check for and install updated requirements
 
 call npm update
 
-pause
+pause

tools/Windows/Modem-Install-Requirements.bat

@@ -0,0 +1,5 @@
+REM Place this batch file in FreeData/freedata_server and then run it
+REM ie. c:\FD-Src\freedata_server
+
+python -m pip install -r ..\requirements.txt
+pause

tools/Windows/Modem-Install-Requrements.bat

@@ -1,5 +0,0 @@
-REM Place this batch file in FreeData/modem and then run it
-REM ie. c:\FD-Src\modem
-
-python -m pip install -r ..\requirements.txt
-pause

tools/Windows/Modem-Launch.bat

@@ -1,10 +1,10 @@
-REM Place this batch file in FreeData/modem and then run it
-REM ie. c:\FD-Src\modem
+REM Place this batch file in FreeData/freedata_server and then run it
+REM ie. c:\FD-Src\freedata_server
 
 REM Set environment variable to let modem know where to find config, change if you need to specify a different config
 set FREEDATA_CONFIG=.\config.ini 
 
-REM launch modem
+REM launch freedata_server
 python server.py
 
-pause
+pause

tools/Windows/Modem-Update-Requirements.bat

@@ -0,0 +1,5 @@
+REM Place this batch file in FreeData/freedata_server and then run it to update requirements
+REM ie. c:\FD-Src\freedata_server
+
+python -m pip install --upgrade -r ..\requirements.txt
+pause

tools/Windows/Modem-Update-Requrements.bat

@@ -1,5 +0,0 @@
-REM Place this batch file in FreeData/modem and then run it to update requirements
-REM ie. c:\FD-Src\modem
-
-python -m pip install --upgrade -r ..\requirements.txt
-pause