mirror of https://github.com/DJ2LS/FreeDATA.git
Compare commits
No commits in common. "main" and "v0.18.1" have entirely different histories.
|
|
@ -53,11 +53,12 @@ jobs:
|
|||
sudo apt update
|
||||
sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf
|
||||
|
||||
- name: Install MacOS dependencies
|
||||
- name: Install MacOS pyAudio
|
||||
if: ${{startsWith(matrix.os, 'macos')}}
|
||||
run: |
|
||||
brew install portaudio
|
||||
python -m pip install --upgrade pip
|
||||
pip3 install pyaudio
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
name: Deploy Python Package
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
|
|
@ -20,64 +17,16 @@ jobs:
|
|||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Install Linux dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install .[build]
|
||||
|
||||
- name: Set package version from tag
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
python3 - <<'EOF'
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
tag = os.environ["RELEASE_TAG"]
|
||||
version = tag.removeprefix("v")
|
||||
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?", version):
|
||||
print(
|
||||
f"::error::Tag '{tag}' does not look like a release version. "
|
||||
f"Use e.g. v1.2.3, v1.2.3-beta, v1.2.3-rc1 or v1.2.3-alpha.1."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
Version(version)
|
||||
except InvalidVersion:
|
||||
print(
|
||||
f"::error::Tag '{tag}' has an unrecognized pre-release suffix "
|
||||
f"('{version}' is not valid PEP 440). Use a standard suffix such "
|
||||
f"as -alpha, -alpha.1, -beta, -rc1 or -dev."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Releasing version {version} (from tag {tag})")
|
||||
|
||||
path = "freedata_server/constants.py"
|
||||
with open(path) as f:
|
||||
content = f.read()
|
||||
|
||||
new_content, count = re.subn(
|
||||
r'^MODEM_VERSION = .*$',
|
||||
f'MODEM_VERSION = "{version}"',
|
||||
content,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
if count != 1:
|
||||
print("::error::Could not find MODEM_VERSION in freedata_server/constants.py")
|
||||
sys.exit(1)
|
||||
|
||||
with open(path, "w") as f:
|
||||
f.write(new_content)
|
||||
EOF
|
||||
grep "^MODEM_VERSION" freedata_server/constants.py
|
||||
|
||||
- name: Build GUI
|
||||
working-directory: freedata_gui
|
||||
run: |
|
||||
|
|
@ -90,7 +39,7 @@ jobs:
|
|||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@v1.14.0
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
with:
|
||||
user: __token__
|
||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||
skip-existing: true
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ ARG HAMLIB_VERSION=4.5.5
|
|||
ENV HAMLIB_VERSION=${HAMLIB_VERSION}
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-colorama wget && \
|
||||
apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-pyaudio python3-colorama wget && \
|
||||
mkdir -p /app/FreeDATA
|
||||
|
||||
WORKDIR /src
|
||||
|
|
|
|||
|
|
@ -89,36 +89,20 @@ def freedv_get_mode_name_by_value(mode: int) -> str:
|
|||
return FREEDV_MODE(mode).name
|
||||
|
||||
|
||||
# Determine the base directory to search for the codec2 shared library.
|
||||
#
|
||||
# In normal (non-frozen) execution this is simply the directory containing
|
||||
# this script, and that's where "lib/codec2/*" lives relative to
|
||||
# freedata_server/codec2.py.
|
||||
#
|
||||
# When compiled by Nuitka into a standalone binary however, data files added
|
||||
# via --include-data-dir/--include-data-files (e.g. "lib=lib") are placed
|
||||
# relative to the *distribution* directory (next to the produced .exe), not
|
||||
# relative to this module's own (nested) package directory. Using
|
||||
# os.path.dirname(__file__) in that case points at "<dist>/freedata_server"
|
||||
# while the actual DLL ends up at "<dist>/lib/codec2/libcodec2.dll" - a
|
||||
# sibling directory, not a child - so the glob below never finds it.
|
||||
#
|
||||
# Nuitka exposes the correct directory via the compiled-only global
|
||||
# `__compiled__.containing_dir`, which always points at the distribution
|
||||
# directory regardless of platform or nesting. See:
|
||||
# https://nuitka.net/user-documentation/common-issue-solutions.html#standalone-finding-files
|
||||
try:
|
||||
script_dir = __compiled__.containing_dir # type: ignore[name-defined]
|
||||
except NameError:
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
# Get the directory of the current script file
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# Use script_dir to construct the paths for file search
|
||||
if sys.platform == "linux":
|
||||
files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*"), recursive=True)
|
||||
files = glob.glob(os.path.join(script_dir, "**/*libcodec2*"), recursive=True)
|
||||
# files.append(os.path.join(script_dir, "libcodec2.so"))
|
||||
elif sys.platform == "darwin":
|
||||
files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dylib"), recursive=True)
|
||||
if hasattr(sys, "_MEIPASS"):
|
||||
files = glob.glob(os.path.join(getattr(sys, "_MEIPASS"), "**/*libcodec2*"), recursive=True)
|
||||
else:
|
||||
files = glob.glob(os.path.join(script_dir, "**/*libcodec2*.dylib"), recursive=True)
|
||||
elif sys.platform in ["win32", "win64"]:
|
||||
files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dll"), recursive=True)
|
||||
files = glob.glob(os.path.join(script_dir, "**\\*libcodec2*.dll"), recursive=True)
|
||||
else:
|
||||
files = []
|
||||
api = None
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import configparser
|
||||
import structlog
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
class CONFIG:
|
||||
|
|
@ -297,11 +296,6 @@ class CONFIG:
|
|||
data if successful, False otherwise.
|
||||
"""
|
||||
try:
|
||||
# need to create the directory before writing to it
|
||||
config_dir = os.path.dirname(self.config_name)
|
||||
if config_dir:
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
|
||||
with open(self.config_name, "w") as configfile:
|
||||
self.parser.write(configfile)
|
||||
self.ctx.config = self.read()
|
||||
|
|
|
|||
|
|
@ -1,33 +1,7 @@
|
|||
# Module for saving some constants
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def _default_app_dir() -> str:
|
||||
"""
|
||||
Per-user directory for config, database and log file, following each
|
||||
OS's own convention rather than forcing a single layout everywhere:
|
||||
- Windows: %APPDATA%\\FreeDATA
|
||||
- macOS: ~/Library/Application Support/FreeDATA
|
||||
- Linux: $XDG_CONFIG_HOME/FreeDATA or ~/.config/FreeDATA
|
||||
Used only when FREEDATA_CONFIG / FREEDATA_DATABASE are not set (e.g. a
|
||||
plain `pip install freedata` run). Keeping this outside the installed
|
||||
package directory means it survives package upgrades/reinstalls.
|
||||
"""
|
||||
home = os.path.expanduser("~")
|
||||
if sys.platform == "win32":
|
||||
base = os.getenv("APPDATA") or home
|
||||
elif sys.platform == "darwin":
|
||||
base = os.path.join(home, "Library", "Application Support")
|
||||
else:
|
||||
base = os.getenv("XDG_CONFIG_HOME") or os.path.join(home, ".config")
|
||||
return os.path.join(base, "FreeDATA")
|
||||
|
||||
|
||||
CONFIG_ENV_VAR = "FREEDATA_CONFIG"
|
||||
DEFAULT_CONFIG_FILE = "config.ini"
|
||||
DEFAULT_APP_DIR = _default_app_dir()
|
||||
MODEM_VERSION = "0.18.2"
|
||||
MODEM_VERSION = "0.18.1"
|
||||
API_VERSION = 4
|
||||
ARQ_PROTOCOL_VERSION = 1
|
||||
LICENSE = "GPL3.0"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from freedata_server.message_system_db_model import Base, Config, Station, Statu
|
|||
import structlog
|
||||
from freedata_server import helpers
|
||||
import os
|
||||
from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION, DEFAULT_APP_DIR
|
||||
from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
|
|
@ -42,19 +42,18 @@ class DatabaseManager:
|
|||
This method determines the database file path based on the
|
||||
environment variable `FREEDATA_DATABASE`. If the variable is set,
|
||||
its value is used as the path. Otherwise, it defaults to
|
||||
`freedata-messages.db` in the per-user app directory
|
||||
(DEFAULT_APP_DIR), so a plain `pip install` keeps the database
|
||||
outside the installed package and it survives upgrades.
|
||||
`freedata-messages.db` in the script directory.
|
||||
|
||||
Returns:
|
||||
str: The database file path as a SQLAlchemy URL.
|
||||
"""
|
||||
script_directory = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
if self.DATABASE_ENV_VAR in os.environ:
|
||||
# db_path = os.getenv(self.DATABASE_ENV_VAR, os.path.join(script_directory, self.DEFAULT_DATABASE_FILE))
|
||||
db_path = os.getenv(self.DATABASE_ENV_VAR)
|
||||
else:
|
||||
db_path = os.path.join(DEFAULT_APP_DIR, self.DEFAULT_DATABASE_FILE)
|
||||
|
||||
os.makedirs(os.path.dirname(db_path), exist_ok=True)
|
||||
db_path = os.path.join(script_directory, self.DEFAULT_DATABASE_FILE)
|
||||
return "sqlite:///" + db_path
|
||||
|
||||
def initialize_default_values(self):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ Created on Wed Dec 23 07:04:24 2020
|
|||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from freedata_server import codec2
|
||||
import numpy as np
|
||||
|
|
@ -71,45 +70,11 @@ class RF:
|
|||
self.AUDIO_STREAMING_CHUNK_SIZE = 2400
|
||||
self.audio_out_queue = queue.Queue()
|
||||
|
||||
# Size of the block the RX DSP chain runs on, in 48 kHz samples. The input
|
||||
# stream is opened with blocksize=0 (PortAudio picks the capture size per
|
||||
# device), so the DSP must not rely on the captured block size:
|
||||
# rx_audio_processing_worker re-blocks whatever the callback is handed
|
||||
# to exactly this size, so a device or setting that delivers some other
|
||||
# size cannot reach the DSP. 4800 (100 ms, 800 samples at 8 kHz) is the
|
||||
# size the rest of the chain is built around:
|
||||
# * codec2.resampler.resample48_to_8 asserts len % FDMDV_OS_48 (6) == 0
|
||||
# and raises AssertionError otherwise,
|
||||
# * audio.calculate_fft pads/truncates to 800 samples at 8 kHz, so a
|
||||
# shorter block is mostly zero padding and the spectrum, channel busy
|
||||
# detection and audio_dbfs all degrade,
|
||||
# * enqueue_streaming_audio_chunks zero-pads every block up to
|
||||
# AUDIO_STREAMING_CHUNK_SIZE, so a shorter block streams mostly
|
||||
# silence and emits one chunk per block regardless of size,
|
||||
# * audio.normalize_audio (rx_auto_audio_level, on by default)
|
||||
# normalizes per block, so a shorter block means a faster, jumpier AGC.
|
||||
# Lowering this lowers RX latency, but needs those four addressed first.
|
||||
self.RX_DSP_BLOCK_48K = 4800
|
||||
|
||||
# Make sure our resampler will work
|
||||
assert (self.AUDIO_SAMPLE_RATE / self.modem_sample_rate) == codec2.api.FDMDV_OS_48 # type: ignore
|
||||
|
||||
self.data_queue_received = queue.Queue()
|
||||
|
||||
# RX audio captured by the real-time sounddevice callback is handed to
|
||||
# rx_audio_processing_worker through this queue, keeping the callback
|
||||
# minimal (copy + enqueue). Running the DSP in the callback under the GIL
|
||||
# is what makes it miss its deadline and overflow on slower CPUs.
|
||||
self.rx_audio_in_queue = queue.Queue(maxsize=100)
|
||||
self.rx_audio_worker_running = False
|
||||
self.rx_audio_worker_thread = None
|
||||
self.rx_audio_dropped_blocks = 0
|
||||
# 48 kHz samples that have been captured but do not yet fill a whole
|
||||
# RX_DSP_BLOCK_48K; they are carried over to the next captured block so the
|
||||
# sample stream handed to the resampler stays gapless (its filter memory
|
||||
# depends on that) and always has a valid length.
|
||||
self.rx_audio_carry_48k = np.empty(0, dtype=np.int16)
|
||||
|
||||
self.demodulator = demodulator.Demodulator(self.ctx)
|
||||
self.modulator = modulator.Modulator(self.ctx)
|
||||
|
||||
|
|
@ -151,12 +116,6 @@ class RF:
|
|||
# self.stream = lambda: None
|
||||
# self.stream.active = False
|
||||
# self.stream.stop
|
||||
# stop the RX audio processing worker before closing the streams
|
||||
self.rx_audio_worker_running = False
|
||||
try:
|
||||
self.rx_audio_in_queue.put_nowait(None)
|
||||
except queue.Full:
|
||||
pass
|
||||
self.sd_input_stream.close()
|
||||
self.sd_output_stream.close()
|
||||
except Exception as e:
|
||||
|
|
@ -204,41 +163,16 @@ class RF:
|
|||
self.resampler = codec2.resampler()
|
||||
|
||||
# SoundDevice audio input stream
|
||||
# blocksize=0 lets PortAudio pick the capture block size per device.
|
||||
# This is deliberate and load-bearing: a fixed blocksize=4800 makes
|
||||
# some virtual devices (measured on snd-aloop) starve/overflow --
|
||||
# a two period ring at 100 ms periods drops audio continuously and
|
||||
# the modem is deaf. latency=0.2 sets the ring depth explicitly so
|
||||
# the ring comes out deep (many small periods) on every device
|
||||
# measured (CM108 hardware and snd-aloop alike).
|
||||
# The capture block size is decoupled from the DSP block size:
|
||||
# PortAudio may deliver any block length here (512/1024-class blocks
|
||||
# are common on real hardware, and are NOT a multiple of codec2's
|
||||
# FDMDV_OS_48 == 6, which resample48_to_8 asserts on).
|
||||
# rx_audio_processing_worker re-blocks whatever arrives into exact
|
||||
# RX_DSP_BLOCK_48K blocks, so no capture size can reach the DSP
|
||||
# chain short or misaligned; see the note on that constant.
|
||||
self.sd_input_stream = sd.InputStream(
|
||||
channels=1,
|
||||
dtype="int16",
|
||||
callback=self.sd_input_audio_callback,
|
||||
device=in_dev_index,
|
||||
samplerate=self.AUDIO_SAMPLE_RATE,
|
||||
blocksize=0,
|
||||
latency=0.2,
|
||||
blocksize=4800,
|
||||
)
|
||||
self.sd_input_stream.start()
|
||||
|
||||
# process RX audio off the real-time callback thread
|
||||
self.rx_audio_carry_48k = np.empty(0, dtype=np.int16) # no stale audio across restarts
|
||||
self.rx_audio_worker_running = True
|
||||
self.rx_audio_worker_thread = threading.Thread(
|
||||
target=self.rx_audio_processing_worker,
|
||||
name="rx_audio_processing_worker",
|
||||
daemon=True,
|
||||
)
|
||||
self.rx_audio_worker_thread.start()
|
||||
|
||||
self.sd_output_stream = sd.OutputStream(
|
||||
channels=1,
|
||||
dtype="int16",
|
||||
|
|
@ -481,97 +415,34 @@ class RF:
|
|||
# if status.input_overflow:
|
||||
# self.self.ctx.modem_service.put("restart")
|
||||
return
|
||||
# Keep this real-time callback minimal: copy the captured block and hand
|
||||
# it to rx_audio_processing_worker. The DSP (resample, FFT, demod-buffer
|
||||
# push) runs there so a long GIL hold by another thread cannot stall this
|
||||
# callback and cause a sounddevice input overflow on slower hardware.
|
||||
try:
|
||||
self.rx_audio_in_queue.put_nowait(indata.copy())
|
||||
except queue.Full:
|
||||
# worker is not draining fast enough; drop this block (counted)
|
||||
self.rx_audio_dropped_blocks += 1
|
||||
audio_48k = np.frombuffer(indata, dtype=np.int16)
|
||||
audio_8k = self.resampler.resample48_to_8(audio_48k)
|
||||
|
||||
def rx_audio_processing_worker(self) -> None:
|
||||
"""Performs all RX audio DSP off the real-time input callback.
|
||||
self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue)
|
||||
|
||||
Drains rx_audio_in_queue (raw 48 kHz int16 blocks copied by
|
||||
sd_input_audio_callback) and runs the resample to 8 kHz, optional
|
||||
level/FFT processing and the demodulator-buffer push on a normal
|
||||
worker thread, so the audio callback is never blocked by these
|
||||
operations (or by the GIL while another thread holds it).
|
||||
"""
|
||||
while self.rx_audio_worker_running:
|
||||
try:
|
||||
indata = self.rx_audio_in_queue.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if indata is None: # shutdown sentinel
|
||||
break
|
||||
try:
|
||||
self.process_rx_audio_block(indata)
|
||||
except Exception as e:
|
||||
self.log.warning("[AUDIO EXCEPTION]", e=e)
|
||||
if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"):
|
||||
audio_8k = audio.normalize_audio(audio_8k)
|
||||
|
||||
def process_rx_audio_block(self, indata) -> None:
|
||||
"""Re-blocks one captured audio block and runs the DSP chain on it.
|
||||
audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level)
|
||||
|
||||
The input stream is free to hand the callback any block size, so the
|
||||
captured samples are appended to rx_audio_carry_48k and the DSP chain is
|
||||
run once per whole RX_DSP_BLOCK_48K available. Anything left over is
|
||||
carried into the next captured block rather than being processed short:
|
||||
a short block would fail codec2's "multiple of 6" resampler assertion and
|
||||
silently degrade the FFT, streaming and AGC paths (see RX_DSP_BLOCK_48K).
|
||||
if not self.ctx.state_manager.isTransmitting():
|
||||
audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager)
|
||||
|
||||
Args:
|
||||
indata (np.ndarray): One captured 48 kHz int16 block, any length.
|
||||
"""
|
||||
captured_48k = np.frombuffer(indata, dtype=np.int16)
|
||||
self.rx_audio_carry_48k = np.concatenate((self.rx_audio_carry_48k, captured_48k))
|
||||
|
||||
block = self.RX_DSP_BLOCK_48K
|
||||
processed = 0
|
||||
while len(self.rx_audio_carry_48k) - processed >= block:
|
||||
self.run_rx_audio_dsp(self.rx_audio_carry_48k[processed : processed + block])
|
||||
processed += block
|
||||
|
||||
if processed:
|
||||
# copy so the carry does not keep the whole concatenated block alive
|
||||
self.rx_audio_carry_48k = self.rx_audio_carry_48k[processed:].copy()
|
||||
|
||||
def run_rx_audio_dsp(self, audio_48k: np.ndarray) -> None:
|
||||
"""Runs the RX DSP chain on exactly one RX_DSP_BLOCK_48K of audio.
|
||||
|
||||
Resamples to 8 kHz, feeds the audio streaming queue, applies the optional
|
||||
auto level and the configured RX gain, updates the FFT, and pushes the
|
||||
result into each decoding demodulator buffer that has room for it.
|
||||
|
||||
Args:
|
||||
audio_48k (np.ndarray): RX_DSP_BLOCK_48K 48 kHz int16 samples.
|
||||
"""
|
||||
audio_8k = self.resampler.resample48_to_8(audio_48k)
|
||||
|
||||
self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue)
|
||||
|
||||
if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"):
|
||||
audio_8k = audio.normalize_audio(audio_8k)
|
||||
|
||||
audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level)
|
||||
|
||||
if not self.ctx.state_manager.isTransmitting():
|
||||
audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager)
|
||||
|
||||
length_audio_8k_level_adjusted = len(audio_8k_level_adjusted)
|
||||
# Avoid buffer overflow by filling only if buffer for
|
||||
# selected datachannel mode is not full
|
||||
index = 0
|
||||
for mode in self.demodulator.MODE_DICT:
|
||||
mode_data = self.demodulator.MODE_DICT[mode]
|
||||
audiobuffer = mode_data["audio_buffer"]
|
||||
decode = mode_data["decode"]
|
||||
index += 1
|
||||
if audiobuffer:
|
||||
if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size:
|
||||
self.demodulator.buffer_overflow_counter[index] += 1
|
||||
self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter)
|
||||
elif decode:
|
||||
audiobuffer.push(audio_8k_level_adjusted)
|
||||
length_audio_8k_level_adjusted = len(audio_8k_level_adjusted)
|
||||
# Avoid buffer overflow by filling only if buffer for
|
||||
# selected datachannel mode is not full
|
||||
index = 0
|
||||
for mode in self.demodulator.MODE_DICT:
|
||||
mode_data = self.demodulator.MODE_DICT[mode]
|
||||
audiobuffer = mode_data["audio_buffer"]
|
||||
decode = mode_data["decode"]
|
||||
index += 1
|
||||
if audiobuffer:
|
||||
if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size:
|
||||
self.demodulator.buffer_overflow_counter[index] += 1
|
||||
self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter)
|
||||
elif decode:
|
||||
audiobuffer.push(audio_8k_level_adjusted)
|
||||
except Exception as e:
|
||||
self.log.warning("[AUDIO EXCEPTION]", status=status, time=time, frames=frames, e=e)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
import threading
|
||||
|
|
@ -11,7 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from freedata_server.log_handler import setup_logging
|
||||
from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, DEFAULT_APP_DIR, API_VERSION
|
||||
from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, API_VERSION
|
||||
from freedata_server.context import AppContext
|
||||
|
||||
from freedata_server.api.general import router as general_router
|
||||
|
|
@ -28,39 +27,18 @@ import uvicorn
|
|||
# --- Resolve config path FIRST (no logger needed yet) ---
|
||||
def resolve_config_path() -> str:
|
||||
"""
|
||||
Determine the configuration file to use.
|
||||
|
||||
Uses FREEDATA_CONFIG if set, otherwise defaults to a per-user config
|
||||
directory (DEFAULT_APP_DIR). If no config file exists yet at that
|
||||
location, a fresh one is bootstrapped from the bundled
|
||||
config.ini.example template so a plain `pip install freedata` followed
|
||||
by `freedata` works out of the box without any manual setup.
|
||||
Determine the configuration file to use (env var or default next to this file).
|
||||
Exits if not found.
|
||||
"""
|
||||
candidate = os.path.abspath(
|
||||
os.getenv(
|
||||
CONFIG_ENV_VAR,
|
||||
os.path.join(DEFAULT_APP_DIR, DEFAULT_CONFIG_FILE),
|
||||
)
|
||||
candidate = os.getenv(
|
||||
CONFIG_ENV_VAR,
|
||||
os.path.join(os.path.dirname(__file__), DEFAULT_CONFIG_FILE),
|
||||
)
|
||||
|
||||
if not os.path.exists(candidate):
|
||||
# We cannot log to file yet since we don't know the directory; write to stderr.
|
||||
template = os.path.join(os.path.dirname(__file__), "config.ini.example")
|
||||
try:
|
||||
os.makedirs(os.path.dirname(candidate), exist_ok=True)
|
||||
if os.path.isfile(template):
|
||||
shutil.copyfile(template, candidate)
|
||||
sys.stderr.write(f"[INFO] No config found - created a default one at: {candidate}\n")
|
||||
else:
|
||||
sys.stderr.write(
|
||||
f"[FATAL] Config file not found and no template available to create one: {candidate}\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"[FATAL] Could not create config file at {candidate}: {e}\n")
|
||||
sys.exit(1)
|
||||
|
||||
return candidate
|
||||
sys.stderr.write(f"[FATAL] Config file not found: {candidate}\n")
|
||||
sys.exit(1)
|
||||
return os.path.abspath(candidate)
|
||||
|
||||
|
||||
config_file = resolve_config_path()
|
||||
|
|
@ -117,15 +95,11 @@ async def nocache(request: Request, call_next):
|
|||
|
||||
|
||||
# Static GUI mounting
|
||||
# Order matters: prefer paths anchored to this file's location (work no
|
||||
# matter what the current working directory is) over cwd-relative
|
||||
# fallbacks kept for backwards compatibility with older layouts.
|
||||
potential_gui_dirs = [
|
||||
os.path.join(os.path.dirname(__file__), "gui"), # nuitka standalone build
|
||||
os.path.join(os.path.dirname(os.path.dirname(__file__)), "freedata_gui", "dist"), # pip install (sibling package)
|
||||
"../freedata_gui/dist",
|
||||
"freedata_gui/dist",
|
||||
"FreeDATA/freedata_gui/dist",
|
||||
os.path.join(os.path.dirname(__file__), "gui"),
|
||||
]
|
||||
gui_dir = next((d for d in potential_gui_dirs if os.path.isdir(d)), None)
|
||||
if gui_dir:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ requires-python = ">=3.10"
|
|||
dependencies = [
|
||||
"numpy",
|
||||
"psutil",
|
||||
"PyAudio",
|
||||
"pyserial",
|
||||
"sounddevice",
|
||||
"structlog",
|
||||
|
|
@ -77,13 +78,12 @@ nuitka = [
|
|||
|
||||
[tool.setuptools.packages.find]
|
||||
where = [ "." ]
|
||||
include = [
|
||||
"freedata_server*",
|
||||
"freedata_gui",
|
||||
exclude = [
|
||||
"tools*",
|
||||
]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
freedata_server = [ "lib/**/*", "config.ini.example" ]
|
||||
freedata_server = [ "lib/**/*" ]
|
||||
freedata_gui = [ "dist/**/*" ]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
numpy
|
||||
psutil
|
||||
PyAudio
|
||||
pyserial
|
||||
sounddevice
|
||||
structlog
|
||||
|
|
|
|||
|
|
@ -1,168 +0,0 @@
|
|||
"""Tests for the real-time-safe RX audio callback / worker split.
|
||||
|
||||
modem.RF.sd_input_audio_callback no longer runs the RX DSP inline; it copies the
|
||||
captured block onto rx_audio_in_queue and returns, and rx_audio_processing_worker
|
||||
drains that queue and runs the DSP (resample 48->8 kHz, FFT, demod-buffer push).
|
||||
|
||||
The stream is opened with blocksize=0, so the captured block size is whatever
|
||||
PortAudio has available: device specific, variable, and not a multiple of
|
||||
anything. rx_audio_processing_worker therefore re-blocks the captured stream to
|
||||
RX_DSP_BLOCK_48K before any DSP runs on it.
|
||||
|
||||
These tests exercise that split directly with synthetic blocks, so they need no
|
||||
audio hardware (CI has none). They cover:
|
||||
- the worker actually performs the relocated DSP on an enqueued block,
|
||||
- odd capture sizes are re-blocked rather than passed to the DSP short, and
|
||||
- the callback drops (and counts) instead of blocking when the queue is full,
|
||||
which is the real-time-safety property the change exists to provide.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from freedata_server.context import AppContext
|
||||
from freedata_server import modem, codec2
|
||||
|
||||
CONFIG = "freedata_server/config.ini.example"
|
||||
|
||||
# A capture size PortAudio really does hand us, and deliberately NOT a multiple of
|
||||
# codec2's FDMDV_OS_48 (6) -- 512 % 6 == 2. Passing this straight to
|
||||
# resample48_to_8 trips its "multiple of 6" assertion, which is what made the RX
|
||||
# chain deaf (every block raising AssertionError) once blocksize=0 was used.
|
||||
CAPTURE_FRAMES = 512
|
||||
|
||||
|
||||
def _rf():
|
||||
"""A real RF wired to a real AppContext, without opening audio devices.
|
||||
|
||||
start_modem() would normally create the resampler (and, in TESTMODE, start
|
||||
the demodulator decode threads); we only need the resampler here, so we set
|
||||
it directly and leave the demod buffers as None -- the worker's buffer-push
|
||||
is guarded by `if audiobuffer` and is intentionally not under test.
|
||||
"""
|
||||
ctx = AppContext(CONFIG)
|
||||
ctx.TESTMODE = True
|
||||
rf = modem.RF(ctx)
|
||||
rf.resampler = codec2.resampler()
|
||||
return rf
|
||||
|
||||
|
||||
def _block(frames=CAPTURE_FRAMES):
|
||||
# sounddevice delivers indata as shape (frames, channels); int16 mono here.
|
||||
return (np.random.randn(frames, 1) * 3000).astype(np.int16)
|
||||
|
||||
|
||||
class TestRxAudioCallbackWorkerSplit(unittest.TestCase):
|
||||
def test_worker_processes_enqueued_blocks(self):
|
||||
"""Blocks handed to the callback are drained and DSP'd by the worker."""
|
||||
rf = _rf()
|
||||
rf.rx_audio_worker_running = True
|
||||
worker = threading.Thread(target=rf.rx_audio_processing_worker, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
# status=None -> the block is enqueued (a truthy status is an
|
||||
# over/underflow and is dropped by the callback, unchanged by this PR).
|
||||
# Feed enough captured blocks to complete at least one DSP block.
|
||||
for _ in range(rf.RX_DSP_BLOCK_48K // CAPTURE_FRAMES + 1):
|
||||
rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None)
|
||||
|
||||
# The worker resamples to 8 kHz and feeds enqueue_streaming_audio_chunks,
|
||||
# which lands on ctx.audio_rx_queue -- our deterministic "DSP ran" signal.
|
||||
deadline = time.time() + 5
|
||||
while rf.ctx.audio_rx_queue.qsize() == 0 and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
|
||||
self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "worker did not process the enqueued RX audio block")
|
||||
self.assertTrue(rf.rx_audio_in_queue.empty(), "worker should have drained the input queue")
|
||||
self.assertEqual(rf.rx_audio_dropped_blocks, 0, "no block should be dropped under normal operation")
|
||||
finally:
|
||||
rf.rx_audio_worker_running = False
|
||||
rf.rx_audio_in_queue.put_nowait(None) # release the worker's get()
|
||||
worker.join(timeout=2)
|
||||
|
||||
def test_callback_drops_and_does_not_block_when_queue_full(self):
|
||||
"""With the worker stalled and the queue full, the callback drops the
|
||||
block (counted) and returns immediately -- it must never block the
|
||||
real-time audio thread."""
|
||||
rf = _rf() # worker intentionally NOT started -> queue never drains
|
||||
for _ in range(rf.rx_audio_in_queue.maxsize):
|
||||
rf.rx_audio_in_queue.put_nowait(object())
|
||||
self.assertTrue(rf.rx_audio_in_queue.full())
|
||||
|
||||
t0 = time.perf_counter()
|
||||
rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None)
|
||||
elapsed = time.perf_counter() - t0
|
||||
|
||||
self.assertEqual(rf.rx_audio_dropped_blocks, 1, "a full queue must drop the block and count it")
|
||||
self.assertLess(elapsed, 0.05, "callback must not block on a full queue (real-time safety)")
|
||||
|
||||
|
||||
class TestRxAudioReblocking(unittest.TestCase):
|
||||
"""The capture block size must never reach the DSP chain.
|
||||
|
||||
blocksize=0 means PortAudio picks the size, and real devices hand back sizes
|
||||
that are not multiples of codec2's FDMDV_OS_48 (6). resample48_to_8 asserts on
|
||||
those, so process_rx_audio_block must accumulate instead of resampling short.
|
||||
These call process_rx_audio_block directly (no worker thread) so a failure is
|
||||
a raised exception rather than a swallowed, logged one.
|
||||
"""
|
||||
|
||||
def test_odd_capture_size_does_not_reach_the_resampler(self):
|
||||
"""A 512-frame capture (512 % 6 == 2) must not raise AssertionError."""
|
||||
rf = _rf()
|
||||
# One short block: not enough for a DSP block, so it is carried, not resampled.
|
||||
rf.process_rx_audio_block(_block(CAPTURE_FRAMES))
|
||||
self.assertEqual(len(rf.rx_audio_carry_48k), CAPTURE_FRAMES)
|
||||
self.assertEqual(rf.ctx.audio_rx_queue.qsize(), 0, "a partial DSP block must not be processed short")
|
||||
|
||||
def test_carry_reassembles_whole_dsp_blocks(self):
|
||||
"""Odd captures are accumulated into exact RX_DSP_BLOCK_48K blocks."""
|
||||
rf = _rf()
|
||||
processed = []
|
||||
rf.run_rx_audio_dsp = lambda audio_48k: processed.append(len(audio_48k))
|
||||
|
||||
# A spread of sizes a real device might deliver, none a multiple of 6.
|
||||
sizes = [512, 1024, 441, 512, 2048, 1024, 512, 940, 512, 1024]
|
||||
for size in sizes:
|
||||
rf.process_rx_audio_block(_block(size))
|
||||
|
||||
total = sum(sizes)
|
||||
self.assertEqual(
|
||||
processed,
|
||||
[rf.RX_DSP_BLOCK_48K] * (total // rf.RX_DSP_BLOCK_48K),
|
||||
"every DSP invocation must get exactly one whole block",
|
||||
)
|
||||
self.assertEqual(len(rf.rx_audio_carry_48k), total % rf.RX_DSP_BLOCK_48K, "remainder must be carried over")
|
||||
|
||||
def test_reblocking_preserves_the_sample_stream(self):
|
||||
"""No sample is dropped, duplicated or reordered by the re-blocking.
|
||||
|
||||
The resampler's filter memory spans blocks, so the stream it sees has to be
|
||||
the captured stream exactly.
|
||||
"""
|
||||
rf = _rf()
|
||||
seen = []
|
||||
rf.run_rx_audio_dsp = lambda audio_48k: seen.append(np.array(audio_48k))
|
||||
|
||||
sizes = [700, 1300, 512, 4800, 441]
|
||||
captured = [np.arange(s, dtype=np.int16).reshape(-1, 1) for s in sizes]
|
||||
for block in captured:
|
||||
rf.process_rx_audio_block(block)
|
||||
|
||||
expected = np.concatenate([b.reshape(-1) for b in captured])
|
||||
got = np.concatenate(seen + [rf.rx_audio_carry_48k])
|
||||
np.testing.assert_array_equal(got, expected)
|
||||
|
||||
def test_real_resampler_accepts_every_reblocked_block(self):
|
||||
"""End to end with the real codec2 resampler: odd captures, no assertion."""
|
||||
rf = _rf()
|
||||
for size in (512, 1024, 441, 2048, 512, 1024, 512, 4800):
|
||||
rf.process_rx_audio_block(_block(size)) # raises AssertionError if short
|
||||
self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "DSP should have run on the reassembled blocks")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -44,9 +44,6 @@
|
|||
#
|
||||
#
|
||||
# Changelog:
|
||||
# 2.10: 24 Jul 2026
|
||||
# Remove python3-pyaudio (unused dependency, FreeDATA uses sounddevice)
|
||||
#
|
||||
# 2.9: 10 Jan Sep 2026
|
||||
# Add Ubuntu 24.10 and 25.04
|
||||
# Change hamlib default version to 4.6.5
|
||||
|
|
@ -167,7 +164,7 @@ case $osname in
|
|||
"Debian GNU/Linux")
|
||||
case $osversion in
|
||||
"11" | "12" | "13")
|
||||
sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev
|
||||
sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev
|
||||
;;
|
||||
|
||||
*)
|
||||
|
|
@ -185,7 +182,7 @@ case $osname in
|
|||
"Ubuntu" | "Linux Mint")
|
||||
case $osversion in
|
||||
"21.3" | "22.04" | "24.04" | "24.10" | "25.04" )
|
||||
sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev
|
||||
sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev
|
||||
;;
|
||||
|
||||
*)
|
||||
|
|
@ -200,7 +197,7 @@ case $osname in
|
|||
"Fedora Linux")
|
||||
case $osversion in
|
||||
"VERSION_ID=40" | "VERSION_ID=41")
|
||||
sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel
|
||||
sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pyaudio python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
|
|
|
|||
Loading…
Reference in New Issue