bump version

v0.18.2

.github/workflows/build_server.yml

@@ -53,12 +53,11 @@ jobs:
           sudo apt update
           sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf
 
-      - name: Install MacOS pyAudio
+      - name: Install MacOS dependencies
         if: ${{startsWith(matrix.os, 'macos')}}
         run: |
           brew install portaudio
           python -m pip install --upgrade pip
-          pip3 install pyaudio
 
       - name: Install Python dependencies
         run: |

.github/workflows/pip_package.yml

@@ -1,5 +1,8 @@
 name: Deploy Python Package
-on: [push]
+on:
+  push:
+    tags:
+      - "v*"
 
 jobs:
   deploy:
@@ -17,16 +20,64 @@ jobs:
       with:
         node-version: 24
 
-    - name: Install Linux dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf
-
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
         pip install .[build]
 
+    - name: Set package version from tag
+      env:
+        RELEASE_TAG: ${{ github.ref_name }}
+      run: |
+        python3 - <<'EOF'
+        import os
+        import re
+        import sys
+
+        from packaging.version import InvalidVersion, Version
+
+        tag = os.environ["RELEASE_TAG"]
+        version = tag.removeprefix("v")
+
+        if not re.fullmatch(r"\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?", version):
+            print(
+                f"::error::Tag '{tag}' does not look like a release version. "
+                f"Use e.g. v1.2.3, v1.2.3-beta, v1.2.3-rc1 or v1.2.3-alpha.1."
+            )
+            sys.exit(1)
+
+        try:
+            Version(version)
+        except InvalidVersion:
+            print(
+                f"::error::Tag '{tag}' has an unrecognized pre-release suffix "
+                f"('{version}' is not valid PEP 440). Use a standard suffix such "
+                f"as -alpha, -alpha.1, -beta, -rc1 or -dev."
+            )
+            sys.exit(1)
+
+        print(f"Releasing version {version} (from tag {tag})")
+
+        path = "freedata_server/constants.py"
+        with open(path) as f:
+            content = f.read()
+
+        new_content, count = re.subn(
+            r'^MODEM_VERSION = .*$',
+            f'MODEM_VERSION = "{version}"',
+            content,
+            count=1,
+            flags=re.MULTILINE,
+        )
+        if count != 1:
+            print("::error::Could not find MODEM_VERSION in freedata_server/constants.py")
+            sys.exit(1)
+
+        with open(path, "w") as f:
+            f.write(new_content)
+        EOF
+        grep "^MODEM_VERSION" freedata_server/constants.py
+
     - name: Build GUI
       working-directory: freedata_gui
       run: |
@@ -39,7 +90,7 @@ jobs:
 
     - name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@v1.14.0
-      if: startsWith(github.ref, 'refs/tags/v')
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
+        skip-existing: true

Dockerfile

@@ -18,7 +18,7 @@ ARG HAMLIB_VERSION=4.5.5
 ENV HAMLIB_VERSION=${HAMLIB_VERSION}
 
 RUN apt-get update && \
-  apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-pyaudio python3-colorama wget && \
+  apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-colorama wget && \
   mkdir -p /app/FreeDATA
 
 WORKDIR /src

freedata_server/codec2.py

@@ -89,20 +89,36 @@ def freedv_get_mode_name_by_value(mode: int) -> str:
     return FREEDV_MODE(mode).name
 
 
-# Get the directory of the current script file
-script_dir = os.path.dirname(os.path.abspath(__file__))
+# Determine the base directory to search for the codec2 shared library.
+#
+# In normal (non-frozen) execution this is simply the directory containing
+# this script, and that's where "lib/codec2/*" lives relative to
+# freedata_server/codec2.py.
+#
+# When compiled by Nuitka into a standalone binary however, data files added
+# via --include-data-dir/--include-data-files (e.g. "lib=lib") are placed
+# relative to the *distribution* directory (next to the produced .exe), not
+# relative to this module's own (nested) package directory. Using
+# os.path.dirname(__file__) in that case points at "<dist>/freedata_server"
+# while the actual DLL ends up at "<dist>/lib/codec2/libcodec2.dll" - a
+# sibling directory, not a child - so the glob below never finds it.
+#
+# Nuitka exposes the correct directory via the compiled-only global
+# `__compiled__.containing_dir`, which always points at the distribution
+# directory regardless of platform or nesting. See:
+# https://nuitka.net/user-documentation/common-issue-solutions.html#standalone-finding-files
+try:
+    script_dir = __compiled__.containing_dir  # type: ignore[name-defined]
+except NameError:
+    script_dir = os.path.dirname(os.path.abspath(__file__))
 
 # Use script_dir to construct the paths for file search
 if sys.platform == "linux":
-    files = glob.glob(os.path.join(script_dir, "**/*libcodec2*"), recursive=True)
-    # files.append(os.path.join(script_dir, "libcodec2.so"))
+    files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*"), recursive=True)
 elif sys.platform == "darwin":
-    if hasattr(sys, "_MEIPASS"):
-        files = glob.glob(os.path.join(getattr(sys, "_MEIPASS"), "**/*libcodec2*"), recursive=True)
-    else:
-        files = glob.glob(os.path.join(script_dir, "**/*libcodec2*.dylib"), recursive=True)
+    files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dylib"), recursive=True)
 elif sys.platform in ["win32", "win64"]:
-    files = glob.glob(os.path.join(script_dir, "**\\*libcodec2*.dll"), recursive=True)
+    files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dll"), recursive=True)
 else:
     files = []
 api = None

freedata_server/config.py

@@ -1,6 +1,7 @@
 import configparser
 import structlog
 import json
+import os
 
 
 class CONFIG:
@@ -296,6 +297,11 @@ class CONFIG:
             data if successful, False otherwise.
         """
         try:
+            # need to create the directory before writing to it
+            config_dir = os.path.dirname(self.config_name)
+            if config_dir:
+                os.makedirs(config_dir, exist_ok=True)
+
             with open(self.config_name, "w") as configfile:
                 self.parser.write(configfile)
                 self.ctx.config = self.read()

freedata_server/constants.py

@@ -1,7 +1,33 @@
 # Module for saving some constants
+import os
+import sys
+
+
+def _default_app_dir() -> str:
+    """
+    Per-user directory for config, database and log file, following each
+    OS's own convention rather than forcing a single layout everywhere:
+      - Windows: %APPDATA%\\FreeDATA
+      - macOS:   ~/Library/Application Support/FreeDATA
+      - Linux:   $XDG_CONFIG_HOME/FreeDATA or ~/.config/FreeDATA
+    Used only when FREEDATA_CONFIG / FREEDATA_DATABASE are not set (e.g. a
+    plain `pip install freedata` run). Keeping this outside the installed
+    package directory means it survives package upgrades/reinstalls.
+    """
+    home = os.path.expanduser("~")
+    if sys.platform == "win32":
+        base = os.getenv("APPDATA") or home
+    elif sys.platform == "darwin":
+        base = os.path.join(home, "Library", "Application Support")
+    else:
+        base = os.getenv("XDG_CONFIG_HOME") or os.path.join(home, ".config")
+    return os.path.join(base, "FreeDATA")
+
+
 CONFIG_ENV_VAR = "FREEDATA_CONFIG"
 DEFAULT_CONFIG_FILE = "config.ini"
-MODEM_VERSION = "0.18.1"
+DEFAULT_APP_DIR = _default_app_dir()
+MODEM_VERSION = "0.18.2"
 API_VERSION = 4
 ARQ_PROTOCOL_VERSION = 1
 LICENSE = "GPL3.0"

freedata_server/message_system_db_manager.py

@@ -5,7 +5,7 @@ from freedata_server.message_system_db_model import Base, Config, Station, Statu
 import structlog
 from freedata_server import helpers
 import os
-from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION
+from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION, DEFAULT_APP_DIR
 
 
 class DatabaseManager:
@@ -42,18 +42,19 @@ class DatabaseManager:
         This method determines the database file path based on the
         environment variable `FREEDATA_DATABASE`. If the variable is set,
         its value is used as the path. Otherwise, it defaults to
-        `freedata-messages.db` in the script directory.
+        `freedata-messages.db` in the per-user app directory
+        (DEFAULT_APP_DIR), so a plain `pip install` keeps the database
+        outside the installed package and it survives upgrades.
 
         Returns:
             str: The database file path as a SQLAlchemy URL.
         """
-        script_directory = os.path.dirname(os.path.abspath(__file__))
-
         if self.DATABASE_ENV_VAR in os.environ:
-            # db_path = os.getenv(self.DATABASE_ENV_VAR, os.path.join(script_directory, self.DEFAULT_DATABASE_FILE))
             db_path = os.getenv(self.DATABASE_ENV_VAR)
         else:
-            db_path = os.path.join(script_directory, self.DEFAULT_DATABASE_FILE)
+            db_path = os.path.join(DEFAULT_APP_DIR, self.DEFAULT_DATABASE_FILE)
+
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
         return "sqlite:///" + db_path
 
     def initialize_default_values(self):

freedata_server/modem.py

@@ -6,6 +6,7 @@ Created on Wed Dec 23 07:04:24 2020
 """
 
 import queue
+import threading
 import time
 from freedata_server import codec2
 import numpy as np
@@ -70,11 +71,45 @@ class RF:
         self.AUDIO_STREAMING_CHUNK_SIZE = 2400
         self.audio_out_queue = queue.Queue()
 
+        # Size of the block the RX DSP chain runs on, in 48 kHz samples. The input
+        # stream is opened with blocksize=0 (PortAudio picks the capture size per
+        # device), so the DSP must not rely on the captured block size:
+        # rx_audio_processing_worker re-blocks whatever the callback is handed
+        # to exactly this size, so a device or setting that delivers some other
+        # size cannot reach the DSP. 4800 (100 ms, 800 samples at 8 kHz) is the
+        # size the rest of the chain is built around:
+        #   * codec2.resampler.resample48_to_8 asserts len % FDMDV_OS_48 (6) == 0
+        #     and raises AssertionError otherwise,
+        #   * audio.calculate_fft pads/truncates to 800 samples at 8 kHz, so a
+        #     shorter block is mostly zero padding and the spectrum, channel busy
+        #     detection and audio_dbfs all degrade,
+        #   * enqueue_streaming_audio_chunks zero-pads every block up to
+        #     AUDIO_STREAMING_CHUNK_SIZE, so a shorter block streams mostly
+        #     silence and emits one chunk per block regardless of size,
+        #   * audio.normalize_audio (rx_auto_audio_level, on by default)
+        #     normalizes per block, so a shorter block means a faster, jumpier AGC.
+        # Lowering this lowers RX latency, but needs those four addressed first.
+        self.RX_DSP_BLOCK_48K = 4800
+
         # Make sure our resampler will work
         assert (self.AUDIO_SAMPLE_RATE / self.modem_sample_rate) == codec2.api.FDMDV_OS_48  # type: ignore
 
         self.data_queue_received = queue.Queue()
 
+        # RX audio captured by the real-time sounddevice callback is handed to
+        # rx_audio_processing_worker through this queue, keeping the callback
+        # minimal (copy + enqueue). Running the DSP in the callback under the GIL
+        # is what makes it miss its deadline and overflow on slower CPUs.
+        self.rx_audio_in_queue = queue.Queue(maxsize=100)
+        self.rx_audio_worker_running = False
+        self.rx_audio_worker_thread = None
+        self.rx_audio_dropped_blocks = 0
+        # 48 kHz samples that have been captured but do not yet fill a whole
+        # RX_DSP_BLOCK_48K; they are carried over to the next captured block so the
+        # sample stream handed to the resampler stays gapless (its filter memory
+        # depends on that) and always has a valid length.
+        self.rx_audio_carry_48k = np.empty(0, dtype=np.int16)
+
         self.demodulator = demodulator.Demodulator(self.ctx)
         self.modulator = modulator.Modulator(self.ctx)
 
@@ -116,6 +151,12 @@ class RF:
             # self.stream = lambda: None
             # self.stream.active = False
             # self.stream.stop
+            # stop the RX audio processing worker before closing the streams
+            self.rx_audio_worker_running = False
+            try:
+                self.rx_audio_in_queue.put_nowait(None)
+            except queue.Full:
+                pass
             self.sd_input_stream.close()
             self.sd_output_stream.close()
         except Exception as e:
@@ -163,16 +204,41 @@ class RF:
             self.resampler = codec2.resampler()
 
             # SoundDevice audio input stream
+            # blocksize=0 lets PortAudio pick the capture block size per device.
+            # This is deliberate and load-bearing: a fixed blocksize=4800 makes
+            # some virtual devices (measured on snd-aloop) starve/overflow --
+            # a two period ring at 100 ms periods drops audio continuously and
+            # the modem is deaf. latency=0.2 sets the ring depth explicitly so
+            # the ring comes out deep (many small periods) on every device
+            # measured (CM108 hardware and snd-aloop alike).
+            # The capture block size is decoupled from the DSP block size:
+            # PortAudio may deliver any block length here (512/1024-class blocks
+            # are common on real hardware, and are NOT a multiple of codec2's
+            # FDMDV_OS_48 == 6, which resample48_to_8 asserts on).
+            # rx_audio_processing_worker re-blocks whatever arrives into exact
+            # RX_DSP_BLOCK_48K blocks, so no capture size can reach the DSP
+            # chain short or misaligned; see the note on that constant.
             self.sd_input_stream = sd.InputStream(
                 channels=1,
                 dtype="int16",
                 callback=self.sd_input_audio_callback,
                 device=in_dev_index,
                 samplerate=self.AUDIO_SAMPLE_RATE,
-                blocksize=4800,
+                blocksize=0,
+                latency=0.2,
             )
             self.sd_input_stream.start()
 
+            # process RX audio off the real-time callback thread
+            self.rx_audio_carry_48k = np.empty(0, dtype=np.int16)  # no stale audio across restarts
+            self.rx_audio_worker_running = True
+            self.rx_audio_worker_thread = threading.Thread(
+                target=self.rx_audio_processing_worker,
+                name="rx_audio_processing_worker",
+                daemon=True,
+            )
+            self.rx_audio_worker_thread.start()
+
             self.sd_output_stream = sd.OutputStream(
                 channels=1,
                 dtype="int16",
@@ -415,34 +481,97 @@ class RF:
             # if status.input_overflow:
             #    self.self.ctx.modem_service.put("restart")
             return
+        # Keep this real-time callback minimal: copy the captured block and hand
+        # it to rx_audio_processing_worker. The DSP (resample, FFT, demod-buffer
+        # push) runs there so a long GIL hold by another thread cannot stall this
+        # callback and cause a sounddevice input overflow on slower hardware.
         try:
-            audio_48k = np.frombuffer(indata, dtype=np.int16)
-            audio_8k = self.resampler.resample48_to_8(audio_48k)
+            self.rx_audio_in_queue.put_nowait(indata.copy())
+        except queue.Full:
+            # worker is not draining fast enough; drop this block (counted)
+            self.rx_audio_dropped_blocks += 1
 
-            self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue)
+    def rx_audio_processing_worker(self) -> None:
+        """Performs all RX audio DSP off the real-time input callback.
 
-            if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"):
-                audio_8k = audio.normalize_audio(audio_8k)
+        Drains rx_audio_in_queue (raw 48 kHz int16 blocks copied by
+        sd_input_audio_callback) and runs the resample to 8 kHz, optional
+        level/FFT processing and the demodulator-buffer push on a normal
+        worker thread, so the audio callback is never blocked by these
+        operations (or by the GIL while another thread holds it).
+        """
+        while self.rx_audio_worker_running:
+            try:
+                indata = self.rx_audio_in_queue.get(timeout=0.5)
+            except queue.Empty:
+                continue
+            if indata is None:  # shutdown sentinel
+                break
+            try:
+                self.process_rx_audio_block(indata)
+            except Exception as e:
+                self.log.warning("[AUDIO EXCEPTION]", e=e)
 
-            audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level)
+    def process_rx_audio_block(self, indata) -> None:
+        """Re-blocks one captured audio block and runs the DSP chain on it.
 
-            if not self.ctx.state_manager.isTransmitting():
-                audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager)
+        The input stream is free to hand the callback any block size, so the
+        captured samples are appended to rx_audio_carry_48k and the DSP chain is
+        run once per whole RX_DSP_BLOCK_48K available. Anything left over is
+        carried into the next captured block rather than being processed short:
+        a short block would fail codec2's "multiple of 6" resampler assertion and
+        silently degrade the FFT, streaming and AGC paths (see RX_DSP_BLOCK_48K).
 
-            length_audio_8k_level_adjusted = len(audio_8k_level_adjusted)
-            # Avoid buffer overflow by filling only if buffer for
-            # selected datachannel mode is not full
-            index = 0
-            for mode in self.demodulator.MODE_DICT:
-                mode_data = self.demodulator.MODE_DICT[mode]
-                audiobuffer = mode_data["audio_buffer"]
-                decode = mode_data["decode"]
-                index += 1
-                if audiobuffer:
-                    if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size:
-                        self.demodulator.buffer_overflow_counter[index] += 1
-                        self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter)
-                    elif decode:
-                        audiobuffer.push(audio_8k_level_adjusted)
-        except Exception as e:
-            self.log.warning("[AUDIO EXCEPTION]", status=status, time=time, frames=frames, e=e)
+        Args:
+            indata (np.ndarray): One captured 48 kHz int16 block, any length.
+        """
+        captured_48k = np.frombuffer(indata, dtype=np.int16)
+        self.rx_audio_carry_48k = np.concatenate((self.rx_audio_carry_48k, captured_48k))
+
+        block = self.RX_DSP_BLOCK_48K
+        processed = 0
+        while len(self.rx_audio_carry_48k) - processed >= block:
+            self.run_rx_audio_dsp(self.rx_audio_carry_48k[processed : processed + block])
+            processed += block
+
+        if processed:
+            # copy so the carry does not keep the whole concatenated block alive
+            self.rx_audio_carry_48k = self.rx_audio_carry_48k[processed:].copy()
+
+    def run_rx_audio_dsp(self, audio_48k: np.ndarray) -> None:
+        """Runs the RX DSP chain on exactly one RX_DSP_BLOCK_48K of audio.
+
+        Resamples to 8 kHz, feeds the audio streaming queue, applies the optional
+        auto level and the configured RX gain, updates the FFT, and pushes the
+        result into each decoding demodulator buffer that has room for it.
+
+        Args:
+            audio_48k (np.ndarray): RX_DSP_BLOCK_48K 48 kHz int16 samples.
+        """
+        audio_8k = self.resampler.resample48_to_8(audio_48k)
+
+        self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue)
+
+        if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"):
+            audio_8k = audio.normalize_audio(audio_8k)
+
+        audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level)
+
+        if not self.ctx.state_manager.isTransmitting():
+            audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager)
+
+        length_audio_8k_level_adjusted = len(audio_8k_level_adjusted)
+        # Avoid buffer overflow by filling only if buffer for
+        # selected datachannel mode is not full
+        index = 0
+        for mode in self.demodulator.MODE_DICT:
+            mode_data = self.demodulator.MODE_DICT[mode]
+            audiobuffer = mode_data["audio_buffer"]
+            decode = mode_data["decode"]
+            index += 1
+            if audiobuffer:
+                if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size:
+                    self.demodulator.buffer_overflow_counter[index] += 1
+                    self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter)
+                elif decode:
+                    audiobuffer.push(audio_8k_level_adjusted)

freedata_server/server.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 
 import threading
@@ -10,7 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 
 from freedata_server.log_handler import setup_logging
-from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, API_VERSION
+from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, DEFAULT_APP_DIR, API_VERSION
 from freedata_server.context import AppContext
 
 from freedata_server.api.general import router as general_router
@@ -27,18 +28,39 @@ import uvicorn
 # --- Resolve config path FIRST (no logger needed yet) ---
 def resolve_config_path() -> str:
     """
-    Determine the configuration file to use (env var or default next to this file).
-    Exits if not found.
+    Determine the configuration file to use.
+
+    Uses FREEDATA_CONFIG if set, otherwise defaults to a per-user config
+    directory (DEFAULT_APP_DIR). If no config file exists yet at that
+    location, a fresh one is bootstrapped from the bundled
+    config.ini.example template so a plain `pip install freedata` followed
+    by `freedata` works out of the box without any manual setup.
     """
-    candidate = os.getenv(
-        CONFIG_ENV_VAR,
-        os.path.join(os.path.dirname(__file__), DEFAULT_CONFIG_FILE),
+    candidate = os.path.abspath(
+        os.getenv(
+            CONFIG_ENV_VAR,
+            os.path.join(DEFAULT_APP_DIR, DEFAULT_CONFIG_FILE),
+        )
     )
+
     if not os.path.exists(candidate):
         # We cannot log to file yet since we don't know the directory; write to stderr.
-        sys.stderr.write(f"[FATAL] Config file not found: {candidate}\n")
-        sys.exit(1)
-    return os.path.abspath(candidate)
+        template = os.path.join(os.path.dirname(__file__), "config.ini.example")
+        try:
+            os.makedirs(os.path.dirname(candidate), exist_ok=True)
+            if os.path.isfile(template):
+                shutil.copyfile(template, candidate)
+                sys.stderr.write(f"[INFO] No config found - created a default one at: {candidate}\n")
+            else:
+                sys.stderr.write(
+                    f"[FATAL] Config file not found and no template available to create one: {candidate}\n"
+                )
+                sys.exit(1)
+        except OSError as e:
+            sys.stderr.write(f"[FATAL] Could not create config file at {candidate}: {e}\n")
+            sys.exit(1)
+
+    return candidate
 
 
 config_file = resolve_config_path()
@@ -95,11 +117,15 @@ async def nocache(request: Request, call_next):
 
 
 # Static GUI mounting
+# Order matters: prefer paths anchored to this file's location (work no
+# matter what the current working directory is) over cwd-relative
+# fallbacks kept for backwards compatibility with older layouts.
 potential_gui_dirs = [
+    os.path.join(os.path.dirname(__file__), "gui"),  # nuitka standalone build
+    os.path.join(os.path.dirname(os.path.dirname(__file__)), "freedata_gui", "dist"),  # pip install (sibling package)
     "../freedata_gui/dist",
     "freedata_gui/dist",
     "FreeDATA/freedata_gui/dist",
-    os.path.join(os.path.dirname(__file__), "gui"),
 ]
 gui_dir = next((d for d in potential_gui_dirs if os.path.isdir(d)), None)
 if gui_dir:

pyproject.toml

@@ -34,7 +34,6 @@ requires-python = ">=3.10"
 dependencies = [
     "numpy",
     "psutil",
-    "PyAudio",
     "pyserial",
     "sounddevice",
     "structlog",
@@ -78,12 +77,13 @@ nuitka = [
 
 [tool.setuptools.packages.find]
 where = [ "." ]
-exclude = [
-    "tools*",
+include = [
+    "freedata_server*",
+    "freedata_gui",
 ]
 
 [tool.setuptools.package-data]
-freedata_server = [ "lib/**/*" ]
+freedata_server = [ "lib/**/*", "config.ini.example" ]
 freedata_gui = [ "dist/**/*" ]
 
 [tool.setuptools.dynamic]

requirements.txt

@@ -1,6 +1,5 @@
 numpy
 psutil
-PyAudio
 pyserial
 sounddevice
 structlog

tests/test_rx_audio_callback.py

@@ -0,0 +1,168 @@
+"""Tests for the real-time-safe RX audio callback / worker split.
+
+modem.RF.sd_input_audio_callback no longer runs the RX DSP inline; it copies the
+captured block onto rx_audio_in_queue and returns, and rx_audio_processing_worker
+drains that queue and runs the DSP (resample 48->8 kHz, FFT, demod-buffer push).
+
+The stream is opened with blocksize=0, so the captured block size is whatever
+PortAudio has available: device specific, variable, and not a multiple of
+anything. rx_audio_processing_worker therefore re-blocks the captured stream to
+RX_DSP_BLOCK_48K before any DSP runs on it.
+
+These tests exercise that split directly with synthetic blocks, so they need no
+audio hardware (CI has none). They cover:
+  - the worker actually performs the relocated DSP on an enqueued block,
+  - odd capture sizes are re-blocked rather than passed to the DSP short, and
+  - the callback drops (and counts) instead of blocking when the queue is full,
+    which is the real-time-safety property the change exists to provide.
+"""
+
+import threading
+import time
+import unittest
+
+import numpy as np
+
+from freedata_server.context import AppContext
+from freedata_server import modem, codec2
+
+CONFIG = "freedata_server/config.ini.example"
+
+# A capture size PortAudio really does hand us, and deliberately NOT a multiple of
+# codec2's FDMDV_OS_48 (6) -- 512 % 6 == 2. Passing this straight to
+# resample48_to_8 trips its "multiple of 6" assertion, which is what made the RX
+# chain deaf (every block raising AssertionError) once blocksize=0 was used.
+CAPTURE_FRAMES = 512
+
+
+def _rf():
+    """A real RF wired to a real AppContext, without opening audio devices.
+
+    start_modem() would normally create the resampler (and, in TESTMODE, start
+    the demodulator decode threads); we only need the resampler here, so we set
+    it directly and leave the demod buffers as None -- the worker's buffer-push
+    is guarded by `if audiobuffer` and is intentionally not under test.
+    """
+    ctx = AppContext(CONFIG)
+    ctx.TESTMODE = True
+    rf = modem.RF(ctx)
+    rf.resampler = codec2.resampler()
+    return rf
+
+
+def _block(frames=CAPTURE_FRAMES):
+    # sounddevice delivers indata as shape (frames, channels); int16 mono here.
+    return (np.random.randn(frames, 1) * 3000).astype(np.int16)
+
+
+class TestRxAudioCallbackWorkerSplit(unittest.TestCase):
+    def test_worker_processes_enqueued_blocks(self):
+        """Blocks handed to the callback are drained and DSP'd by the worker."""
+        rf = _rf()
+        rf.rx_audio_worker_running = True
+        worker = threading.Thread(target=rf.rx_audio_processing_worker, daemon=True)
+        worker.start()
+        try:
+            # status=None -> the block is enqueued (a truthy status is an
+            # over/underflow and is dropped by the callback, unchanged by this PR).
+            # Feed enough captured blocks to complete at least one DSP block.
+            for _ in range(rf.RX_DSP_BLOCK_48K // CAPTURE_FRAMES + 1):
+                rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None)
+
+            # The worker resamples to 8 kHz and feeds enqueue_streaming_audio_chunks,
+            # which lands on ctx.audio_rx_queue -- our deterministic "DSP ran" signal.
+            deadline = time.time() + 5
+            while rf.ctx.audio_rx_queue.qsize() == 0 and time.time() < deadline:
+                time.sleep(0.02)
+
+            self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "worker did not process the enqueued RX audio block")
+            self.assertTrue(rf.rx_audio_in_queue.empty(), "worker should have drained the input queue")
+            self.assertEqual(rf.rx_audio_dropped_blocks, 0, "no block should be dropped under normal operation")
+        finally:
+            rf.rx_audio_worker_running = False
+            rf.rx_audio_in_queue.put_nowait(None)  # release the worker's get()
+            worker.join(timeout=2)
+
+    def test_callback_drops_and_does_not_block_when_queue_full(self):
+        """With the worker stalled and the queue full, the callback drops the
+        block (counted) and returns immediately -- it must never block the
+        real-time audio thread."""
+        rf = _rf()  # worker intentionally NOT started -> queue never drains
+        for _ in range(rf.rx_audio_in_queue.maxsize):
+            rf.rx_audio_in_queue.put_nowait(object())
+        self.assertTrue(rf.rx_audio_in_queue.full())
+
+        t0 = time.perf_counter()
+        rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None)
+        elapsed = time.perf_counter() - t0
+
+        self.assertEqual(rf.rx_audio_dropped_blocks, 1, "a full queue must drop the block and count it")
+        self.assertLess(elapsed, 0.05, "callback must not block on a full queue (real-time safety)")
+
+
+class TestRxAudioReblocking(unittest.TestCase):
+    """The capture block size must never reach the DSP chain.
+
+    blocksize=0 means PortAudio picks the size, and real devices hand back sizes
+    that are not multiples of codec2's FDMDV_OS_48 (6). resample48_to_8 asserts on
+    those, so process_rx_audio_block must accumulate instead of resampling short.
+    These call process_rx_audio_block directly (no worker thread) so a failure is
+    a raised exception rather than a swallowed, logged one.
+    """
+
+    def test_odd_capture_size_does_not_reach_the_resampler(self):
+        """A 512-frame capture (512 % 6 == 2) must not raise AssertionError."""
+        rf = _rf()
+        # One short block: not enough for a DSP block, so it is carried, not resampled.
+        rf.process_rx_audio_block(_block(CAPTURE_FRAMES))
+        self.assertEqual(len(rf.rx_audio_carry_48k), CAPTURE_FRAMES)
+        self.assertEqual(rf.ctx.audio_rx_queue.qsize(), 0, "a partial DSP block must not be processed short")
+
+    def test_carry_reassembles_whole_dsp_blocks(self):
+        """Odd captures are accumulated into exact RX_DSP_BLOCK_48K blocks."""
+        rf = _rf()
+        processed = []
+        rf.run_rx_audio_dsp = lambda audio_48k: processed.append(len(audio_48k))
+
+        # A spread of sizes a real device might deliver, none a multiple of 6.
+        sizes = [512, 1024, 441, 512, 2048, 1024, 512, 940, 512, 1024]
+        for size in sizes:
+            rf.process_rx_audio_block(_block(size))
+
+        total = sum(sizes)
+        self.assertEqual(
+            processed,
+            [rf.RX_DSP_BLOCK_48K] * (total // rf.RX_DSP_BLOCK_48K),
+            "every DSP invocation must get exactly one whole block",
+        )
+        self.assertEqual(len(rf.rx_audio_carry_48k), total % rf.RX_DSP_BLOCK_48K, "remainder must be carried over")
+
+    def test_reblocking_preserves_the_sample_stream(self):
+        """No sample is dropped, duplicated or reordered by the re-blocking.
+
+        The resampler's filter memory spans blocks, so the stream it sees has to be
+        the captured stream exactly.
+        """
+        rf = _rf()
+        seen = []
+        rf.run_rx_audio_dsp = lambda audio_48k: seen.append(np.array(audio_48k))
+
+        sizes = [700, 1300, 512, 4800, 441]
+        captured = [np.arange(s, dtype=np.int16).reshape(-1, 1) for s in sizes]
+        for block in captured:
+            rf.process_rx_audio_block(block)
+
+        expected = np.concatenate([b.reshape(-1) for b in captured])
+        got = np.concatenate(seen + [rf.rx_audio_carry_48k])
+        np.testing.assert_array_equal(got, expected)
+
+    def test_real_resampler_accepts_every_reblocked_block(self):
+        """End to end with the real codec2 resampler: odd captures, no assertion."""
+        rf = _rf()
+        for size in (512, 1024, 441, 2048, 512, 1024, 512, 4800):
+            rf.process_rx_audio_block(_block(size))  # raises AssertionError if short
+        self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "DSP should have run on the reassembled blocks")
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/Linux/install-freedata-linux.sh

@@ -44,6 +44,9 @@
 #
 #
 # Changelog:
+# 2.10:	24 Jul 2026
+#	Remove python3-pyaudio (unused dependency, FreeDATA uses sounddevice)
+#
 # 2.9:	10 Jan Sep 2026
 #	Add Ubuntu 24.10 and 25.04
 #	Change hamlib default version to 4.6.5
@@ -164,7 +167,7 @@ case $osname in
    "Debian GNU/Linux")
 	case $osversion in
 	   "11" | "12" | "13")
-		sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev
+		sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev
 	   ;;
 
 	   *)
@@ -182,7 +185,7 @@ case $osname in
    "Ubuntu" | "Linux Mint")
 	case $osversion in
 	   "21.3" | "22.04" | "24.04" | "24.10" | "25.04" )
-		sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev
+		sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev
 	   ;;
 
 	   *)
@@ -197,7 +200,7 @@ case $osname in
    "Fedora Linux")
 	case $osversion in
 	   "VERSION_ID=40" | "VERSION_ID=41")
-		sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pyaudio python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel
+		sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel
 	   ;;
 	esac
    ;;