Added rngit media conversion
Mark Qvist
parent
602d52f178
commit
27910f25a1
|
|
@ -0,0 +1,164 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# - The Software shall not be used in any kind of system which includes amongst
|
||||
# its functions the ability to purposefully do harm to human beings.
|
||||
#
|
||||
# - The Software shall not be used, directly or indirectly, in the creation of
|
||||
# an artificial intelligence, machine learning or language model training
|
||||
# dataset, including but not limited to any use that contributes to the
|
||||
# training or development of such a model or algorithm.
|
||||
#
|
||||
# - The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import os
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
import RNS
|
||||
|
||||
CONVERSION_TIMEOUT = 8
|
||||
|
||||
# WebP encoding backends, in preference order. All supported
|
||||
# encoder families handle the common still formats (PNG, JPEG,
|
||||
# TIFF, BMP, GIF, WebP).
|
||||
#
|
||||
# Environment overrides:
|
||||
# RNGIT_MEDIA_BACKEND force a specific backend
|
||||
|
||||
BACKENDS = ( ("magick", ["magick", "-", "webp:-"]),
|
||||
("convert", ["convert", "-", "webp:-"]),
|
||||
("gm", ["gm", "convert", "-", "webp:-"]),
|
||||
("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]),
|
||||
("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) )
|
||||
|
||||
_ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND")
|
||||
_winner = None
|
||||
_no_backend_logged = False
|
||||
|
||||
def available_backends():
|
||||
out = []
|
||||
for name, argv in BACKENDS: out.append((name, shutil.which(argv[0]) is not None))
|
||||
return out
|
||||
|
||||
def _selected_backend():
|
||||
global _winner
|
||||
if _ENV_BACKEND:
|
||||
for name, argv in BACKENDS:
|
||||
if name == _ENV_BACKEND and shutil.which(argv[0]) is not None:
|
||||
return (name, argv)
|
||||
return None
|
||||
|
||||
order = list(BACKENDS)
|
||||
if _winner is not None: order.sort(key=lambda backend: backend[0] != _winner)
|
||||
|
||||
for name, argv in order:
|
||||
if shutil.which(argv[0]) is not None:
|
||||
_winner = name
|
||||
return (name, argv)
|
||||
|
||||
return None
|
||||
|
||||
def _terminate(proc):
|
||||
try: proc.kill()
|
||||
except Exception: pass
|
||||
try: proc.wait()
|
||||
except Exception: pass
|
||||
|
||||
def _stderr_tail(proc, limit=1024):
|
||||
try:
|
||||
data = proc.stderr.read(limit)
|
||||
return data.decode("utf-8", "replace").strip()
|
||||
except Exception: return ""
|
||||
|
||||
def _webp_info(data):
|
||||
if len(data) < 30 or data[:4] != b"RIFF" or data[8:12] != b"WEBP": return None
|
||||
fourcc = data[12:16]
|
||||
if fourcc == b"VP8X":
|
||||
width = int.from_bytes(data[24:27], "little") + 1
|
||||
height = int.from_bytes(data[27:30], "little") + 1
|
||||
elif fourcc == b"VP8 ":
|
||||
width = int.from_bytes(data[26:28], "little") & 0x3FFF
|
||||
height = int.from_bytes(data[28:30], "little") & 0x3FFF
|
||||
elif fourcc == b"VP8L":
|
||||
bits = int.from_bytes(data[21:25], "little")
|
||||
width = (bits & 0x3FFF) + 1
|
||||
height = ((bits >> 14) & 0x3FFF) + 1
|
||||
else: return None
|
||||
if width > 0 and height > 0: return (width, height)
|
||||
return None
|
||||
|
||||
def _valid_webp(path):
|
||||
try:
|
||||
with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None
|
||||
except Exception: return False
|
||||
|
||||
def convert_to_webp(input_argv, output_path, cwd=None, timeout=None):
|
||||
global _no_backend_logged
|
||||
if timeout is None: timeout = CONVERSION_TIMEOUT
|
||||
|
||||
backend = _selected_backend()
|
||||
if backend is None:
|
||||
if not _no_backend_logged:
|
||||
_no_backend_logged = True
|
||||
RNS.log("No WebP encoding backend available for media conversion. You can Install ImageMagick or ffmpeg to enable image conversion.", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
backend_name, encoder_argv = backend
|
||||
_no_backend_logged = False
|
||||
|
||||
try:
|
||||
with open(output_path, "wb") as output_fh:
|
||||
input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_proc.stdout, stdout=output_fh, stderr=subprocess.PIPE)
|
||||
|
||||
# The parent must not retain a copy of the input pipe's write
|
||||
# end, or the input process would never see EOF.
|
||||
input_proc.stdout.close()
|
||||
|
||||
deadline = time.time() + timeout
|
||||
try:
|
||||
encoder_proc.wait(timeout=max(0.0, deadline - time.time()))
|
||||
input_proc.wait(timeout=max(0.0, deadline - time.time()))
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate(input_proc)
|
||||
_terminate(encoder_proc)
|
||||
RNS.log(f"Media conversion via {backend_name} timed out after {timeout} seconds", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
if encoder_proc.returncode != 0:
|
||||
detail = ""
|
||||
encoder_tail = _stderr_tail(encoder_proc)
|
||||
input_tail = _stderr_tail(input_proc)
|
||||
if encoder_tail: detail += f" Encoder: {encoder_tail}"
|
||||
if input_tail: detail += f" Input: {input_tail}"
|
||||
RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
if not _valid_webp(output_path):
|
||||
RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
RNS.log(f"Media converted to WebP with {backend_name}", RNS.LOG_DEBUG)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"Error during media conversion: {e}", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
|
@ -32,13 +32,14 @@ import os
|
|||
import time
|
||||
import threading
|
||||
import subprocess
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import RNS
|
||||
import struct
|
||||
import base64
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
from RNS.Utilities.rngit import APP_NAME
|
||||
from RNS.Utilities.rngit import APP_NAME, media
|
||||
from RNS.Utilities.rngit.util import MarkdownToMicron, san_sha
|
||||
from RNS.Utilities.rngit.highlight import SyntaxHighlighter
|
||||
from RNS.vendor.configobj import ConfigObj
|
||||
|
|
@ -50,6 +51,7 @@ from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_sign
|
|||
class NomadNetworkNode():
|
||||
APP_NAME = "nomadnetwork"
|
||||
JOBS_INTERVAL = 5
|
||||
LINK_CLEAN_INTERVAL = 60
|
||||
|
||||
PATH_INDEX = "/page/index.mu"
|
||||
PATH_GROUP = "/page/group.mu"
|
||||
|
|
@ -155,10 +157,13 @@ class NomadNetworkNode():
|
|||
self.templates["no_ident"] = DEFAULT_NO_IDENT_TEMPLATE
|
||||
self.templatesdir = self.owner.configdir+"/templates"
|
||||
self.use_nerdfonts = self.USE_NERDFONTS
|
||||
self.media_conversion = True
|
||||
self.highlight_syntax = True
|
||||
self.highlighter = SyntaxHighlighter()
|
||||
self.mdc = MarkdownToMicron(max_width=self.MAX_RENDER_WIDTH, syntax_highlighter=self.highlighter)
|
||||
self.thanks_deque = deque(maxlen=256)
|
||||
self.active_links = {}
|
||||
self.last_link_clean = 0
|
||||
|
||||
if not os.path.isdir(self.templatesdir):
|
||||
try: os.makedirs(self.templatesdir)
|
||||
|
|
@ -167,6 +172,8 @@ class NomadNetworkNode():
|
|||
if "pages" in self.owner.config:
|
||||
if "unicode_icons" in self.owner.config["pages"]:
|
||||
if self.owner.config["pages"].as_bool("unicode_icons"): self.use_nerdfonts = False
|
||||
if "media_conversion" in self.owner.config["pages"]:
|
||||
self.media_conversion = self.owner.config["pages"].as_bool("media_conversion")
|
||||
|
||||
self.destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, self.APP_NAME, "node")
|
||||
self.destination.set_link_established_callback(self.remote_connected)
|
||||
|
|
@ -213,6 +220,10 @@ class NomadNetworkNode():
|
|||
try:
|
||||
if self.announce_interval and time.time() > self.last_announce + self.announce_interval: self.announce()
|
||||
|
||||
if time.time() > self.last_link_clean + self.LINK_CLEAN_INTERVAL:
|
||||
self.clean_links()
|
||||
self.last_link_clean = time.time()
|
||||
|
||||
except Exception as e: RNS.log(f"Error while running periodic jobs: {e}", RNS.LOG_ERROR)
|
||||
|
||||
def get_announce_app_data(self): return self.node_name.encode("utf-8")
|
||||
|
|
@ -1775,8 +1786,6 @@ class NomadNetworkNode():
|
|||
return False
|
||||
|
||||
comps = media_path.removeprefix("/media").lstrip("/").split("/")
|
||||
RNS.log(comps)
|
||||
|
||||
if len(comps) < 4:
|
||||
RNS.log(f"Insufficient path components in media request", RNS.LOG_DEBUG)
|
||||
return False
|
||||
|
|
@ -1788,8 +1797,6 @@ class NomadNetworkNode():
|
|||
file_path = urllib.parse.unquote_plus(file_path)
|
||||
file_name = os.path.basename(file_path)
|
||||
|
||||
RNS.log(f"{group_name}, {repo_name}, {ref}, {file_path}, {file_name}")
|
||||
|
||||
repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
|
||||
if not repo:
|
||||
RNS.log(f"Repository not found or no access for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_DEBUG)
|
||||
|
|
@ -1812,10 +1819,15 @@ class NomadNetworkNode():
|
|||
return False
|
||||
|
||||
else:
|
||||
stream = self.get_blob_stream(repo_path, resolved_ref, file_path)
|
||||
if stream is not None:
|
||||
return [stream, {"name": file_name.encode("utf-8")}]
|
||||
stream = None
|
||||
response_name = file_name
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if self.media_conversion and file_ext in self.IMAGE_EXTS and file_ext != ".webp":
|
||||
converted = self.get_webp_stream(repo_path, resolved_ref, file_path, link_id)
|
||||
if converted: stream, response_name = converted
|
||||
|
||||
if not stream: stream = self.get_blob_stream(repo_path, resolved_ref, file_path)
|
||||
if stream: return [stream, {"name": response_name.encode("utf-8")}]
|
||||
else:
|
||||
RNS.log(f"Could not resolve blob stream for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
|
||||
return None
|
||||
|
|
@ -2191,6 +2203,40 @@ class NomadNetworkNode():
|
|||
|
||||
return None
|
||||
|
||||
def get_webp_stream(self, repo_path, ref, file_path, link_id):
|
||||
file_path = file_path.strip("/")
|
||||
link = self.active_links.get(link_id)
|
||||
if not link:
|
||||
RNS.log(f"Could not resolve link for media conversion of {file_path}", RNS.LOG_WARNING)
|
||||
return None
|
||||
|
||||
if not hasattr(link, "temporary_directories"): link.temporary_directories = []
|
||||
tmpdir = tempfile.TemporaryDirectory()
|
||||
link.temporary_directories.append(tmpdir)
|
||||
|
||||
stem = os.path.splitext(os.path.basename(file_path))[0]
|
||||
response_name = stem + ".webp"
|
||||
|
||||
try:
|
||||
fd, spool_path = tempfile.mkstemp(prefix=stem+".", suffix=".webp", dir=tmpdir.name)
|
||||
os.close(fd)
|
||||
|
||||
converted = media.convert_to_webp(["git", "show", f"{ref}:{file_path}"], spool_path, cwd=repo_path)
|
||||
if not converted:
|
||||
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
|
||||
tmpdir.cleanup()
|
||||
return None
|
||||
|
||||
spool = open(spool_path, "rb")
|
||||
return spool, response_name
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"Error during media conversion of {file_path} for {link}: {e}", RNS.LOG_WARNING)
|
||||
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
|
||||
try: tmpdir.cleanup()
|
||||
except Exception: pass
|
||||
return None
|
||||
|
||||
def get_refs_info(self, repo_path, default_branch=None):
|
||||
refs = {"heads": [], "tags": []}
|
||||
|
||||
|
|
@ -2802,11 +2848,38 @@ class NomadNetworkNode():
|
|||
|
||||
def remote_connected(self, link):
|
||||
RNS.log(f"Peer connected to {self.destination}", RNS.LOG_DEBUG)
|
||||
self.active_links[link.link_id] = link
|
||||
link.set_remote_identified_callback(self.remote_identified)
|
||||
link.set_link_closed_callback(self.remote_disconnected)
|
||||
|
||||
def remote_disconnected(self, link):
|
||||
RNS.log(f"Peer disconnected from {self.destination}", RNS.LOG_DEBUG)
|
||||
if link.link_id in self.active_links: self.active_links.pop(link.link_id)
|
||||
self.cleanup_link_temporary_resources(link)
|
||||
|
||||
def clean_links(self):
|
||||
stale_links = []
|
||||
for link_id, link in self.active_links.items():
|
||||
if not link.status == RNS.Link.ACTIVE: stale_links.append(link_id)
|
||||
|
||||
cleaned_links = 0
|
||||
for link_id in stale_links:
|
||||
link = self.active_links.pop(link_id, None)
|
||||
if link:
|
||||
self.cleanup_link_temporary_resources(link)
|
||||
cleaned_links += 1
|
||||
|
||||
if cleaned_links > 0: RNS.log(f"Cleaned {cleaned_links} stale link{'s' if cleaned_links != 1 else ''}", RNS.LOG_DEBUG)
|
||||
|
||||
def cleanup_link_temporary_resources(self, link):
|
||||
if hasattr(link, "temporary_directories"):
|
||||
for tmpdir in link.temporary_directories:
|
||||
try:
|
||||
tmpdir.cleanup()
|
||||
RNS.log(f"Cleaned up {tmpdir.name}", RNS.LOG_DEBUG)
|
||||
except Exception as e: RNS.log(f"Error while cleaning temporary directory: {e}", RNS.LOG_ERROR)
|
||||
|
||||
link.temporary_directories = []
|
||||
|
||||
def remote_identified(self, link, identity):
|
||||
RNS.log(f"Peer identified as {link.get_remote_identity()} on {link}", RNS.LOG_DEBUG)
|
||||
|
|
|
|||
|
|
@ -5001,14 +5001,26 @@ internal = rw:9710b86ba12c42d1d8f30f74fe509286
|
|||
|
||||
# serve_nomadnet = no
|
||||
|
||||
# It is possible to disable Nerd Font icons and instead
|
||||
# It is possible to disable Nerd Font icons and instead
|
||||
# use simpler (but more compatible) unicode icons.
|
||||
|
||||
# unicode_icons = yes
|
||||
|
||||
# You can configure whether the page server should try
|
||||
# to convert media files to WebP on the fly, for serving
|
||||
# to nomadnet clients. Enabled by default, but will
|
||||
# require an available encoding backend installed on
|
||||
# your system. Supported backends utilities are "magick",
|
||||
# "convert", "gm", "ffmpeg" and "avconv". If any one is
|
||||
# installed, rngit will auto-detect and use it, but you
|
||||
# can force a specific backend with the environment
|
||||
# variable RNGIT_MEDIA_BACKEND.
|
||||
|
||||
# media_conversion = yes
|
||||
|
||||
|
||||
[logging]
|
||||
# Valid log levels are 0 through 7:
|
||||
# Valid log levels are 0 through 8:
|
||||
# 0: Log only critical information
|
||||
# 1: Log errors and lower log levels
|
||||
# 2: Log warnings and lower log levels
|
||||
|
|
@ -5016,7 +5028,8 @@ internal = rw:9710b86ba12c42d1d8f30f74fe509286
|
|||
# 4: Log info and lower (this is the default)
|
||||
# 5: Verbose logging
|
||||
# 6: Debug logging
|
||||
# 7: Extreme logging
|
||||
# 7: Pathing logging
|
||||
# 8: Extreme logging
|
||||
|
||||
loglevel = 4
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue