From 27910f25a1d028359851fb8f512254cccfece7e3 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 6 Sep 2026 17:39:07 +0200 Subject: [PATCH] Added rngit media conversion --- RNS/Utilities/rngit/media.py | 164 ++++++++++++++++++++++++++++++++++ RNS/Utilities/rngit/pages.py | 89 ++++++++++++++++-- RNS/Utilities/rngit/server.py | 19 +++- 3 files changed, 261 insertions(+), 11 deletions(-) create mode 100644 RNS/Utilities/rngit/media.py diff --git a/RNS/Utilities/rngit/media.py b/RNS/Utilities/rngit/media.py new file mode 100644 index 00000000..af7b7d0b --- /dev/null +++ b/RNS/Utilities/rngit/media.py @@ -0,0 +1,164 @@ +# Reticulum License +# +# Copyright (c) 2016-2026 Mark Qvist +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# - The Software shall not be used in any kind of system which includes amongst +# its functions the ability to purposefully do harm to human beings. +# +# - The Software shall not be used, directly or indirectly, in the creation of +# an artificial intelligence, machine learning or language model training +# dataset, including but not limited to any use that contributes to the +# training or development of such a model or algorithm. +# +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import time +import shutil +import subprocess +import RNS + +CONVERSION_TIMEOUT = 8 + +# WebP encoding backends, in preference order. All supported +# encoder families handle the common still formats (PNG, JPEG, +# TIFF, BMP, GIF, WebP). +# +# Environment overrides: +# RNGIT_MEDIA_BACKEND force a specific backend + +BACKENDS = ( ("magick", ["magick", "-", "webp:-"]), + ("convert", ["convert", "-", "webp:-"]), + ("gm", ["gm", "convert", "-", "webp:-"]), + ("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]), + ("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) ) + +_ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND") +_winner = None +_no_backend_logged = False + +def available_backends(): + out = [] + for name, argv in BACKENDS: out.append((name, shutil.which(argv[0]) is not None)) + return out + +def _selected_backend(): + global _winner + if _ENV_BACKEND: + for name, argv in BACKENDS: + if name == _ENV_BACKEND and shutil.which(argv[0]) is not None: + return (name, argv) + return None + + order = list(BACKENDS) + if _winner is not None: order.sort(key=lambda backend: backend[0] != _winner) + + for name, argv in order: + if shutil.which(argv[0]) is not None: + _winner = name + return (name, argv) + + return None + +def _terminate(proc): + try: proc.kill() + except Exception: pass + try: proc.wait() + except Exception: pass + +def _stderr_tail(proc, limit=1024): + try: + data = proc.stderr.read(limit) + return data.decode("utf-8", "replace").strip() + except Exception: return "" + +def _webp_info(data): + if len(data) < 30 or data[:4] != b"RIFF" or data[8:12] != b"WEBP": return None + fourcc = data[12:16] + if fourcc == b"VP8X": + width = int.from_bytes(data[24:27], "little") + 1 + height = int.from_bytes(data[27:30], "little") + 1 + elif fourcc == b"VP8 ": + width = int.from_bytes(data[26:28], "little") & 0x3FFF + height = int.from_bytes(data[28:30], "little") & 0x3FFF + elif fourcc == b"VP8L": + bits = int.from_bytes(data[21:25], "little") + width = (bits & 0x3FFF) + 1 + height = ((bits >> 14) & 0x3FFF) + 1 + else: return None + if width > 0 and height > 0: return (width, height) + return None + +def _valid_webp(path): + try: + with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None + except Exception: return False + +def convert_to_webp(input_argv, output_path, cwd=None, timeout=None): + global _no_backend_logged + if timeout is None: timeout = CONVERSION_TIMEOUT + + backend = _selected_backend() + if backend is None: + if not _no_backend_logged: + _no_backend_logged = True + RNS.log("No WebP encoding backend available for media conversion. You can Install ImageMagick or ffmpeg to enable image conversion.", RNS.LOG_WARNING) + return False + + backend_name, encoder_argv = backend + _no_backend_logged = False + + try: + with open(output_path, "wb") as output_fh: + input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + encoder_proc = subprocess.Popen(encoder_argv, stdin=input_proc.stdout, stdout=output_fh, stderr=subprocess.PIPE) + + # The parent must not retain a copy of the input pipe's write + # end, or the input process would never see EOF. + input_proc.stdout.close() + + deadline = time.time() + timeout + try: + encoder_proc.wait(timeout=max(0.0, deadline - time.time())) + input_proc.wait(timeout=max(0.0, deadline - time.time())) + except subprocess.TimeoutExpired: + _terminate(input_proc) + _terminate(encoder_proc) + RNS.log(f"Media conversion via {backend_name} timed out after {timeout} seconds", RNS.LOG_WARNING) + return False + + if encoder_proc.returncode != 0: + detail = "" + encoder_tail = _stderr_tail(encoder_proc) + input_tail = _stderr_tail(input_proc) + if encoder_tail: detail += f" Encoder: {encoder_tail}" + if input_tail: detail += f" Input: {input_tail}" + RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING) + return False + + if not _valid_webp(output_path): + RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING) + return False + + RNS.log(f"Media converted to WebP with {backend_name}", RNS.LOG_DEBUG) + return True + + except Exception as e: + RNS.log(f"Error during media conversion: {e}", RNS.LOG_WARNING) + return False diff --git a/RNS/Utilities/rngit/pages.py b/RNS/Utilities/rngit/pages.py index 9e1bb89c..e85f8a45 100644 --- a/RNS/Utilities/rngit/pages.py +++ b/RNS/Utilities/rngit/pages.py @@ -32,13 +32,14 @@ import os import time import threading import subprocess +import tempfile import urllib.parse import RNS import struct import base64 from collections import deque from datetime import datetime -from RNS.Utilities.rngit import APP_NAME +from RNS.Utilities.rngit import APP_NAME, media from RNS.Utilities.rngit.util import MarkdownToMicron, san_sha from RNS.Utilities.rngit.highlight import SyntaxHighlighter from RNS.vendor.configobj import ConfigObj @@ -50,6 +51,7 @@ from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_sign class NomadNetworkNode(): APP_NAME = "nomadnetwork" JOBS_INTERVAL = 5 + LINK_CLEAN_INTERVAL = 60 PATH_INDEX = "/page/index.mu" PATH_GROUP = "/page/group.mu" @@ -155,10 +157,13 @@ class NomadNetworkNode(): self.templates["no_ident"] = DEFAULT_NO_IDENT_TEMPLATE self.templatesdir = self.owner.configdir+"/templates" self.use_nerdfonts = self.USE_NERDFONTS + self.media_conversion = True self.highlight_syntax = True self.highlighter = SyntaxHighlighter() self.mdc = MarkdownToMicron(max_width=self.MAX_RENDER_WIDTH, syntax_highlighter=self.highlighter) self.thanks_deque = deque(maxlen=256) + self.active_links = {} + self.last_link_clean = 0 if not os.path.isdir(self.templatesdir): try: os.makedirs(self.templatesdir) @@ -167,6 +172,8 @@ class NomadNetworkNode(): if "pages" in self.owner.config: if "unicode_icons" in self.owner.config["pages"]: if self.owner.config["pages"].as_bool("unicode_icons"): self.use_nerdfonts = False + if "media_conversion" in self.owner.config["pages"]: + self.media_conversion = self.owner.config["pages"].as_bool("media_conversion") self.destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, self.APP_NAME, "node") self.destination.set_link_established_callback(self.remote_connected) @@ -213,6 +220,10 @@ class NomadNetworkNode(): try: if self.announce_interval and time.time() > self.last_announce + self.announce_interval: self.announce() + if time.time() > self.last_link_clean + self.LINK_CLEAN_INTERVAL: + self.clean_links() + self.last_link_clean = time.time() + except Exception as e: RNS.log(f"Error while running periodic jobs: {e}", RNS.LOG_ERROR) def get_announce_app_data(self): return self.node_name.encode("utf-8") @@ -1775,8 +1786,6 @@ class NomadNetworkNode(): return False comps = media_path.removeprefix("/media").lstrip("/").split("/") - RNS.log(comps) - if len(comps) < 4: RNS.log(f"Insufficient path components in media request", RNS.LOG_DEBUG) return False @@ -1788,8 +1797,6 @@ class NomadNetworkNode(): file_path = urllib.parse.unquote_plus(file_path) file_name = os.path.basename(file_path) - RNS.log(f"{group_name}, {repo_name}, {ref}, {file_path}, {file_name}") - repo = self.get_accessible_repository(remote_identity, group_name, repo_name) if not repo: RNS.log(f"Repository not found or no access for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_DEBUG) @@ -1812,10 +1819,15 @@ class NomadNetworkNode(): return False else: - stream = self.get_blob_stream(repo_path, resolved_ref, file_path) - if stream is not None: - return [stream, {"name": file_name.encode("utf-8")}] + stream = None + response_name = file_name + file_ext = os.path.splitext(file_path)[1].lower() + if self.media_conversion and file_ext in self.IMAGE_EXTS and file_ext != ".webp": + converted = self.get_webp_stream(repo_path, resolved_ref, file_path, link_id) + if converted: stream, response_name = converted + if not stream: stream = self.get_blob_stream(repo_path, resolved_ref, file_path) + if stream: return [stream, {"name": response_name.encode("utf-8")}] else: RNS.log(f"Could not resolve blob stream for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING) return None @@ -2191,6 +2203,40 @@ class NomadNetworkNode(): return None + def get_webp_stream(self, repo_path, ref, file_path, link_id): + file_path = file_path.strip("/") + link = self.active_links.get(link_id) + if not link: + RNS.log(f"Could not resolve link for media conversion of {file_path}", RNS.LOG_WARNING) + return None + + if not hasattr(link, "temporary_directories"): link.temporary_directories = [] + tmpdir = tempfile.TemporaryDirectory() + link.temporary_directories.append(tmpdir) + + stem = os.path.splitext(os.path.basename(file_path))[0] + response_name = stem + ".webp" + + try: + fd, spool_path = tempfile.mkstemp(prefix=stem+".", suffix=".webp", dir=tmpdir.name) + os.close(fd) + + converted = media.convert_to_webp(["git", "show", f"{ref}:{file_path}"], spool_path, cwd=repo_path) + if not converted: + if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir) + tmpdir.cleanup() + return None + + spool = open(spool_path, "rb") + return spool, response_name + + except Exception as e: + RNS.log(f"Error during media conversion of {file_path} for {link}: {e}", RNS.LOG_WARNING) + if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir) + try: tmpdir.cleanup() + except Exception: pass + return None + def get_refs_info(self, repo_path, default_branch=None): refs = {"heads": [], "tags": []} @@ -2802,11 +2848,38 @@ class NomadNetworkNode(): def remote_connected(self, link): RNS.log(f"Peer connected to {self.destination}", RNS.LOG_DEBUG) + self.active_links[link.link_id] = link link.set_remote_identified_callback(self.remote_identified) link.set_link_closed_callback(self.remote_disconnected) def remote_disconnected(self, link): RNS.log(f"Peer disconnected from {self.destination}", RNS.LOG_DEBUG) + if link.link_id in self.active_links: self.active_links.pop(link.link_id) + self.cleanup_link_temporary_resources(link) + + def clean_links(self): + stale_links = [] + for link_id, link in self.active_links.items(): + if not link.status == RNS.Link.ACTIVE: stale_links.append(link_id) + + cleaned_links = 0 + for link_id in stale_links: + link = self.active_links.pop(link_id, None) + if link: + self.cleanup_link_temporary_resources(link) + cleaned_links += 1 + + if cleaned_links > 0: RNS.log(f"Cleaned {cleaned_links} stale link{'s' if cleaned_links != 1 else ''}", RNS.LOG_DEBUG) + + def cleanup_link_temporary_resources(self, link): + if hasattr(link, "temporary_directories"): + for tmpdir in link.temporary_directories: + try: + tmpdir.cleanup() + RNS.log(f"Cleaned up {tmpdir.name}", RNS.LOG_DEBUG) + except Exception as e: RNS.log(f"Error while cleaning temporary directory: {e}", RNS.LOG_ERROR) + + link.temporary_directories = [] def remote_identified(self, link, identity): RNS.log(f"Peer identified as {link.get_remote_identity()} on {link}", RNS.LOG_DEBUG) diff --git a/RNS/Utilities/rngit/server.py b/RNS/Utilities/rngit/server.py index 69610b53..9320db64 100644 --- a/RNS/Utilities/rngit/server.py +++ b/RNS/Utilities/rngit/server.py @@ -5001,14 +5001,26 @@ internal = rw:9710b86ba12c42d1d8f30f74fe509286 # serve_nomadnet = no -# It is possible to disable Nerd Font icons and instead +# It is possible to disable Nerd Font icons and instead # use simpler (but more compatible) unicode icons. # unicode_icons = yes +# You can configure whether the page server should try +# to convert media files to WebP on the fly, for serving +# to nomadnet clients. Enabled by default, but will +# require an available encoding backend installed on +# your system. Supported backends utilities are "magick", +# "convert", "gm", "ffmpeg" and "avconv". If any one is +# installed, rngit will auto-detect and use it, but you +# can force a specific backend with the environment +# variable RNGIT_MEDIA_BACKEND. + +# media_conversion = yes + [logging] -# Valid log levels are 0 through 7: +# Valid log levels are 0 through 8: # 0: Log only critical information # 1: Log errors and lower log levels # 2: Log warnings and lower log levels @@ -5016,7 +5028,8 @@ internal = rw:9710b86ba12c42d1d8f30f74fe509286 # 4: Log info and lower (this is the default) # 5: Verbose logging # 6: Debug logging -# 7: Extreme logging +# 7: Pathing logging +# 8: Extreme logging loglevel = 4