ARIA/iris_x_hermes
1
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases 4 Wiki Activity

M4: media upload/download/playback (both directions)

Browse Source 
Gateway plugin:
- media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff ->
  cache_*_from_bytes -> media.upload.ack
- media.offer / media.pull (chunked) for agent-sent media, delivery-path
  security re-checked at pull time
- send_* overrides mint media_id and emit media.offer
- message.send media_refs resolve to cached inbound media
- per-send + per-chunk timeouts so a stalled peer can't starve the rest

App (Kotlin CMP):
- Protocol: media frame types/payloads/builders
- GatewayClient: binary session, uploadMedia (chunked + streaming sha256),
  pullMedia serialized via Mutex so concurrent offers don't interleave
- ChatStore/IrisController: MediaItem, attachments, auto-pull on offer
- Platform media: SAF picker, ExoPlayer (audio mini-player + video), image
  loader, FileProvider document open (Android); AWT-free desktop actuals
- ChatScreen: attach button + chips, media rendering, keyboard dismiss on send

UI polish:
- preserve image aspect ratio (no stretching), cap dominant dimension
- adjustResize so only chat content squeezes for the keyboard
- clear focus (hide keyboard) on send

Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json +
07-media.md; M4 marked complete in 14-milestones.md.

Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
ARIA committed 2026-08-19 17:29:39 +02:00
1 parent 60296b33fe
commit 913ee91024
28 files changed
+2234 -53
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

gateway-plugin/adapter.py
+318 -9
View File
@@ -18,8 +18,17 @@ Milestone M2: agent transparency. ``send()``/``edit_message()`` are mapped to
``message.start``/``message.update``/``message.stop`` (streaming), tool
progress is classified into structured ``tool.start``/``tool.end`` frames,
interim commentary becomes ``commentary`` frames, and the code-style
reasoning prefix is split into a ``reasoning`` field. Media, outbox, push,
and search land in later milestones (see ``docs/14-milestones.md``).
reasoning prefix is split into a ``reasoning`` field. Outbox and search land
in M3; media, push, and desktop land in later milestones (see
``docs/14-milestones.md``).
Milestone M4: media. Inbound ``media.upload`` (chunked binary frames) is
reassembled in a temp file, verified (size + sha256), re-sniffed, and cached
via hermes ``cache_*_from_bytes``; the resulting refs attach to the next
``message.send`` as ``MessageEvent.media_urls``. Outbound ``send_*`` calls
register the (delivery-validated) file in the media registry and emit
``media.offer``; ``media.pull`` streams the file back as chunked binary
frames, re-checking ``validate_media_delivery_path`` at pull time.
Configuration in config.yaml::
@@ -87,10 +96,12 @@ from gateway.platforms.base import ( # noqa: E402
SendResult,
MessageEvent,
MessageType,
validate_media_delivery_path,
)
from gateway.config import Platform # noqa: E402
from hermes_constants import get_hermes_home # noqa: E402
from . import media as media_bridge # noqa: E402
from . import protocol # noqa: E402
from . import search as search_bridge # noqa: E402
from .channels import get_directory # noqa: E402
@@ -678,6 +689,10 @@ class AndroidAdapter(BasePlatformAdapter):
get_hermes_home() / "android" / "outbox.db",
retention_hours=self.outbox_retention_hours,
)
# M4: media registry (inbound upload refs + outbound offers) and the
# last finalized assistant message id per chat (offer association).
self._media = media_bridge.MediaStore(get_hermes_home())
self._last_message_id: Dict[str, str] = {}
def _turn_state(self, chat_id: str) -> _TurnState:
st = self._turns.get(chat_id)
@@ -841,6 +856,8 @@ class AndroidAdapter(BasePlatformAdapter):
ts=int(time.time() * 1000),
),
)
# M4: media offers emitted after this final associate with it.
self._last_message_id[chat_id] = message_id
await self._close_open_tool(chat_id, state, thread_id)
self._reset_tool_state(state)
state.active = False
@@ -901,6 +918,8 @@ class AndroidAdapter(BasePlatformAdapter):
ts=int(time.time() * 1000),
),
)
# M4: media offers emitted after this final associate with it.
self._last_message_id[chat_id] = message_id
await self._close_open_tool(chat_id, state, thread_id)
self._reset_tool_state(state)
state.active = False
@@ -1049,6 +1068,49 @@ class AndroidAdapter(BasePlatformAdapter):
"""Clear the typing indicator (``typing`` frame, on=false)."""
await self._ws_server.broadcast(protocol.typing(chat_id, False))
# ── M4: outbound media (agent -> app) ─────────────────────────────────
#
# The gateway's dispatch partition (gateway/run.py) extracts MEDIA: tags /
# image URLs from the final response, filters them through
# ``filter_media_delivery_paths``, then calls these ``send_*`` overrides
# with local file paths. We re-validate each path (defense in depth),
# register it in the media registry, mint a ``media_id``, and emit
# ``media.offer``; the app fetches the bytes via ``media.pull``.
async def _offer_media(
self,
chat_id: str,
path: str,
kind: str,
filename: Optional[str],
metadata: Optional[Dict[str, Any]],
) -> SendResult:
safe = validate_media_delivery_path(path)
if safe is None:
logger.warning("android: media path failed delivery validation: %s", path)
return SendResult(success=False, error="android: media path not deliverable")
try:
size = os.path.getsize(safe)
except OSError as e:
logger.warning("android: media file unreadable %s: %s", safe, e)
return SendResult(success=False, error="android: media file unreadable")
entry = self._media.register_outbound(
safe, kind, media_bridge.mime_for_path(safe), filename or os.path.basename(safe), size
)
thread_id = _thread_id_from_metadata(metadata)
frame = protocol.media_offer(
entry.media_id,
entry.kind,
entry.mime,
entry.size,
entry.filename,
chat_id=chat_id,
thread_id=thread_id,
message_id=self._last_message_id.get(chat_id),
)
await self._broadcast_or_log(chat_id, frame)
return SendResult(success=True, message_id=entry.media_id)
async def send_image(
self,
chat_id: str,
@@ -1057,8 +1119,63 @@ class AndroidAdapter(BasePlatformAdapter):
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> SendResult:
"""Send an image. M1: not implemented (M4)."""
return SendResult(success=False, error="android: media not implemented yet (M4)")
"""Send an image (M4: local files offered over WS; remote URLs fall
back to the base text rendering)."""
if image_url.startswith("file://"):
from urllib.parse import unquote
return await self._offer_media(chat_id, unquote(image_url[7:]), "image", None, metadata)
return await super().send_image(
chat_id, image_url, caption=caption, reply_to=reply_to, metadata=metadata
)
async def send_image_file(
self,
chat_id: str,
image_path: str,
caption: Optional[str] = None,
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> SendResult:
"""Send a local image file (M4)."""
return await self._offer_media(chat_id, image_path, "image", None, metadata)
async def send_video(
self,
chat_id: str,
video_path: str,
caption: Optional[str] = None,
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> SendResult:
"""Send a video (M4)."""
return await self._offer_media(chat_id, video_path, "video", None, metadata)
async def send_voice(
self,
chat_id: str,
audio_path: str,
caption: Optional[str] = None,
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> SendResult:
"""Send a voice note / audio file (M4)."""
return await self._offer_media(chat_id, audio_path, "voice", None, metadata)
async def send_document(
self,
chat_id: str,
file_path: str,
caption: Optional[str] = None,
file_name: Optional[str] = None,
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> SendResult:
"""Send a document (M4)."""
return await self._offer_media(chat_id, file_path, "document", file_name, metadata)
# ── Inbound (app -> agent) ────────────────────────────────────────────
@@ -1068,10 +1185,24 @@ class AndroidAdapter(BasePlatformAdapter):
Echoes the user message to all devices (multi-device sync + ack),
then builds a ``MessageEvent`` and hands it to ``handle_message()``
(the gateway's command pipeline + agent turn).
M4: ``media_refs`` reference completed ``media.upload``s; they are
resolved to ``MessageEvent.media_urls``/``media_types`` (local paths
the agent's vision/audio tools can read) and echoed in the user
message's ``media[]`` so every device renders the attachments.
"""
payload = frame.payload
text = payload.get("text")
if not isinstance(text, str) or not text.strip():
text = text if isinstance(text, str) else ""
refs_raw = payload.get("media_refs")
media_refs = (
[r for r in refs_raw if isinstance(r, str) and r]
if isinstance(refs_raw, list)
else []
)
if not text.strip() and not media_refs:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, "message.send requires non-empty text", id=frame.id),
@@ -1091,6 +1222,30 @@ class AndroidAdapter(BasePlatformAdapter):
if not isinstance(reply_to, str) or not reply_to.strip():
reply_to = None
# M4: resolve media refs (single-use; unknown ref -> error).
media_urls: List[str] = []
media_types: List[str] = []
media_wire: List[Dict[str, Any]] = []
for ref in media_refs:
entry = self._media.get_inbound(ref)
if entry is None:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, f"unknown media_ref {ref}", id=frame.id),
)
return
media_urls.append(entry.path)
media_types.append(entry.mime)
media_wire.append(
{
"media_id": entry.media_id,
"kind": entry.kind,
"mime": entry.mime,
"size": entry.size,
"filename": entry.filename,
}
)
device = self._devices.get(device_id) or {}
user_name = device.get("name") or device_id
@@ -1103,10 +1258,31 @@ class AndroidAdapter(BasePlatformAdapter):
role=protocol.ROLE_USER,
text=text,
thread_id=thread_id,
media=media_wire or None,
reply_to=reply_to,
ts=int(time.time() * 1000),
)
await self._ws_server.broadcast(echo)
# Refs are consumed by this message (no replay).
for ref in media_refs:
self._media.pop_inbound(ref)
# M4: a new user turn starts -- stale offer association is dropped.
self._last_message_id.pop(chat_id, None)
kind = media_wire[0]["kind"] if media_wire else None
if kind == "image":
message_type = MessageType.PHOTO
elif kind == "video":
message_type = MessageType.VIDEO
elif kind == "audio":
message_type = MessageType.AUDIO
elif kind == "voice":
message_type = MessageType.VOICE
elif kind == "document":
message_type = MessageType.DOCUMENT
else:
message_type = MessageType.TEXT
source = self.build_source(
chat_id=chat_id,
@@ -1118,15 +1294,143 @@ class AndroidAdapter(BasePlatformAdapter):
)
event = MessageEvent(
text=text,
message_type=MessageType.TEXT,
message_type=message_type,
user_id=device_id,
user_name=user_name,
source=source,
message_id=message_id,
reply_to_message_id=reply_to,
media_urls=media_urls,
media_types=media_types,
)
await self.handle_message(event)
# ── M4: inbound media (app -> agent) ──────────────────────────────────
#
# ``media.upload.start`` -> raw binary frames (one at a time per
# connection) -> ``media.upload.end``. The session streams to a temp
# file (bounded RAM); on end we verify size + sha256, re-sniff the kind,
# and cache via hermes ``cache_*_from_bytes``. ``media.pull`` serves an
# outbound offer as chunked binary frames, re-checking the delivery-path
# validation at pull time.
async def on_media_upload_start(self, frame: protocol.Frame, device_id: str) -> None:
payload = frame.payload
media_ref = str(payload.get("media_ref") or "").strip()
if not media_ref or len(media_ref) > 64:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.start requires media_ref", id=frame.id),
)
return
kind = payload.get("kind")
if kind not in media_bridge.KINDS:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, f"unsupported media kind {kind!r}", id=frame.id),
)
return
mime = str(payload.get("mime") or "application/octet-stream")[:128]
filename = str(payload.get("filename") or "upload")[:255]
size = payload.get("size")
try:
size = int(size) if size is not None else -1
except (TypeError, ValueError):
size = -1
if size <= 0:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.start requires a positive size", id=frame.id),
)
return
if size > self.max_upload_bytes:
await self._ws_server.send_to(
device_id,
protocol.error(
protocol.ERR_MEDIA_TOO_LARGE,
f"upload of {size} bytes exceeds limit ({self.max_upload_bytes})",
id=frame.id,
),
)
return
try:
self._media.create_upload(
device_id, media_ref, kind, mime, filename, size, frame.id,
self.max_upload_bytes,
)
except media_bridge.MediaError as e:
await self._ws_server.send_to(device_id, protocol.error(e.code, e.message, id=frame.id))
return
# No ack: WS ordering guarantees the server processes this before the
# first binary chunk; failures arrive as ``error`` frames.
async def on_media_chunk(self, device_id: str, chunk: bytes) -> None:
session = self._media.get_upload(device_id)
if session is None:
return # stray binary frame: ignore (forward-compat)
session.feed(chunk)
if session.failed:
await self._ws_server.send_to(
device_id,
protocol.error(session.error_code, session.error_message, id=session.request_id),
)
self._media.discard_upload(device_id, session.media_ref)
async def on_media_upload_end(self, frame: protocol.Frame, device_id: str) -> None:
payload = frame.payload
media_ref = str(payload.get("media_ref") or "").strip()
sha256 = str(payload.get("sha256") or "").strip().lower()
if not media_ref:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.end requires media_ref", id=frame.id),
)
return
try:
entry = self._media.complete_upload(device_id, media_ref, sha256)
except media_bridge.MediaError as e:
await self._ws_server.send_to(device_id, protocol.error(e.code, e.message, id=frame.id))
return
await self._ws_server.send_to(
device_id, protocol.media_upload_ack(True, entry.media_id, id=frame.id)
)
async def on_media_pull(self, frame: protocol.Frame, device_id: str) -> None:
payload = frame.payload
media_id = str(payload.get("media_id") or "").strip()
entry = self._media.get_outbound(media_id) if media_id else None
if entry is None:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_NOT_FOUND, f"unknown media_id {media_id!r}", id=frame.id),
)
return
# Delivery-path security: re-validate at pull time (the file may have
# moved / been replaced since the offer).
safe = validate_media_delivery_path(entry.path)
if safe is None:
await self._ws_server.send_to(
device_id,
protocol.error(protocol.ERR_NOT_FOUND, "media no longer deliverable", id=frame.id),
)
return
conn = self._ws_server.connection(device_id)
if conn is None:
return
try:
await media_bridge.stream_file(conn.ws, safe, media_bridge.DEFAULT_CHUNK_BYTES)
except Exception as e:
logger.warning("android: media.pull stream failed for %s: %s", media_id, e)
await self._ws_server.send_to(
device_id, protocol.error(protocol.ERR_INTERNAL, f"pull failed: {e}", id=frame.id)
)
return
await self._ws_server.send_to(device_id, protocol.media_pull_end(True, id=frame.id))
def on_connection_closed(self, device_id: str) -> None:
"""M4: drop in-flight upload temp files for a disconnected device."""
self._media.discard_device(device_id)
# ── M3: channel directory management (app -> agent) ───────────────────
#
# Each request is answered by broadcasting the matching ``channel.*``
@@ -1318,12 +1622,12 @@ class AndroidAdapter(BasePlatformAdapter):
# ── hello.ack helpers ─────────────────────────────────────────────────
def server_caps(self) -> Dict[str, Any]:
"""Capability flags advertised in ``hello.ack`` (M3 surface)."""
"""Capability flags advertised in ``hello.ack`` (M4 surface)."""
return {
"streaming": True, # M2: message.start/update/stop
"reasoning": True, # M2: reasoning field on message / message.stop
"tools": True, # M2: tool.start/progress/end
"media": False, # M4
"media": True, # M4: media.upload/offer/pull
"search": True, # M3: search frame
"push": self.push_backend,
"pickers": False, # M2+
@@ -1431,6 +1735,11 @@ def register(ctx):
"(Android/Desktop). It renders Markdown, inline code, images, "
"audio and video, and shows your reasoning and tool activity. "
"Conversations are organized into channels and optional threads. "
"Keep formatting rich but readable."
"Keep formatting rich but readable. "
"You can send media files natively: to deliver a file to the user, "
"include MEDIA:/absolute/path/to/file in your response. Images "
"(.png, .jpg, .webp) appear as photos, audio (.ogg, .mp3, .m4a) "
"plays inline, videos (.mp4, .webm, .mov) play inline, and other "
"files arrive as downloadable documents."
),
)
gateway-plugin/media.py
+440 -8
View File
@@ -2,15 +2,447 @@
Inbound: ``media.upload`` (chunked binary frames) -> ``cache_*_from_bytes``
-> a ``media_ref`` the adapter attaches to the ``MessageEvent``. Enforces
size limit + sha256 + MIME re-sniff.
size limit + sha256 + MIME re-sniff (the client's declared kind is not
trusted). Chunks are streamed to a temp file so RAM stays bounded during
the transfer; the file is read once at the end for the hermes cache
helpers.
Outbound: ``send_*`` -> stage the file in the media cache, mint a
``media_id``, emit ``media.offer {media_id, mime, size, filename, kind}``;
serve bytes on ``media.pull`` as chunked binary frames. Delivery-path
security via ``validate_media_delivery_path``.
Outbound: ``send_*`` -> register the (already delivery-validated) file in
the media registry, mint a ``media_id``, emit ``media.offer {media_id,
kind, mime, size, filename}``; serve bytes on ``media.pull`` as chunked
binary frames. Delivery-path security via ``validate_media_delivery_path``
(re-checked at pull time, so only allowed files are servable).
Reuses hermes ``cache_image/audio/video/document_from_bytes`` where possible.
All paths under ``get_hermes_home()/"android"/media``.
Reuses hermes ``cache_image/audio/video/document_from_bytes`` + the
``_looks_like_image`` / ``sniff_container`` magic-byte sniffers. Temp files
live under ``get_hermes_home()/"android"/media/tmp``.
Milestone M4.
"""
"""
import asyncio
import hashlib
import logging
import os
import re
import tempfile
import threading
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Optional, Tuple
from gateway.platforms.base import (
_looks_like_image,
cache_audio_from_bytes,
cache_document_from_bytes,
cache_image_from_bytes,
cache_video_from_bytes,
)
from tools.audio_container import sniff_container
logger = logging.getLogger(__name__)
# One WS binary frame carries at most this many media bytes (docs/07 §7.5).
DEFAULT_CHUNK_BYTES = 256 * 1024
KINDS = ("image", "audio", "video", "document", "voice")
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
# Magic-byte containers that are unambiguously audio (vs video-in-same-box).
_AUDIO_CONTAINERS = {"m4a", "ogg", "flac", "wav", "mp3", "aac"}
_VIDEO_CONTAINERS = {"mp4", "webm"}
# Extension -> MIME for outbound offers (the app picks a player/viewer from it).
_EXT_TO_MIME: Dict[str, str] = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".webp": "image/webp",
".gif": "image/gif",
".bmp": "image/bmp",
".heic": "image/heic",
".mp4": "video/mp4",
".mov": "video/quicktime",
".webm": "video/webm",
".mkv": "video/x-matroska",
".avi": "video/x-msvideo",
".3gp": "video/3gpp",
".ogg": "audio/ogg",
".opus": "audio/ogg",
".m4a": "audio/mp4",
".mp3": "audio/mpeg",
".wav": "audio/wav",
".flac": "audio/flac",
".aac": "audio/aac",
".pdf": "application/pdf",
".txt": "text/plain",
".md": "text/markdown",
".csv": "text/csv",
".html": "text/html",
".json": "application/json",
".zip": "application/zip",
".doc": "application/msword",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".xls": "application/vnd.ms-excel",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".ppt": "application/vnd.ms-powerpoint",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
}
# MIME -> extension for inbound caching (the cache helpers take an ext).
_MIME_TO_EXT: Dict[str, str] = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"image/bmp": ".bmp",
"image/heic": ".heic",
"video/mp4": ".mp4",
"video/quicktime": ".mov",
"video/webm": ".webm",
"video/x-matroska": ".mkv",
"video/x-msvideo": ".avi",
"audio/ogg": ".ogg",
"audio/mp4": ".m4a",
"audio/mpeg": ".mp3",
"audio/wav": ".wav",
"audio/flac": ".flac",
"audio/aac": ".aac",
}
class MediaError(Exception):
"""Media flow failure carrying a wire ``error`` code."""
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = code
self.message = message
def mime_for_path(path: str) -> str:
ext = os.path.splitext(path)[1].lower()
return _EXT_TO_MIME.get(ext, "application/octet-stream")
def ext_for_mime(mime: str, filename: str, default: str) -> str:
"""Best-effort extension for the hermes cache helpers.
Prefers the MIME map, falls back to the filename's own extension, then
*default*.
"""
base = (mime or "").split(";")[0].strip().lower()
ext = _MIME_TO_EXT.get(base)
if ext:
return ext
file_ext = os.path.splitext(filename or "")[1].lower()
if file_ext and len(file_ext) <= 6:
return file_ext
return default
def reclassify_kind(head: bytes, declared_kind: str) -> str:
"""Re-sniff the real kind from magic bytes (don't trust the client).
Images are detected first (RIFF/WEBP would otherwise look like a
container); then AV containers. Video declared by the client is kept
when no magic matches (some codecs have no leading magic we check);
anything else falls back to ``document``.
"""
if _looks_like_image(head):
return "image"
container = sniff_container(head)
if container in _AUDIO_CONTAINERS:
return "voice" if declared_kind == "voice" else "audio"
if container in _VIDEO_CONTAINERS:
return "video"
return declared_kind if declared_kind == "video" else "document"
@dataclass
class MediaEntry:
"""One servable/cached media object (inbound ref or outbound offer)."""
media_id: str # mu_* (inbound ref, app-chosen) or md_* (outbound, minted)
kind: str
mime: str
size: int
filename: str
path: str # absolute local path
ts: float = 0.0
class UploadSession:
"""One in-flight ``media.upload`` (per connection; one at a time).
Chunks are appended to a temp file (bounded RAM); the sha256 is
accumulated incrementally. The declared size is enforced as bytes
arrive so an over-limit transfer is rejected early.
"""
def __init__(
self,
media_ref: str,
kind: str,
mime: str,
filename: str,
declared_size: int,
request_id: Optional[int],
max_bytes: int,
tmp_dir: Path,
):
self.media_ref = media_ref
self.kind = kind
self.mime = mime
self.filename = filename
self.declared_size = declared_size
self.request_id = request_id
self.max_bytes = max_bytes
self.received = 0
self.failed = False
self.error_code = ""
self.error_message = ""
self._sha = hashlib.sha256()
fd, self.tmp_path = tempfile.mkstemp(prefix="upl_", dir=str(tmp_dir))
self._fh = os.fdopen(fd, "wb")
def feed(self, chunk: bytes) -> None:
if self.failed:
return
self.received += len(chunk)
if self.max_bytes and self.received > self.max_bytes:
self._fail(
"media_too_large",
f"upload exceeds limit ({self.received} > {self.max_bytes} bytes)",
)
return
try:
self._fh.write(chunk)
self._fh.flush()
self._sha.update(chunk)
except OSError as e:
self._fail("internal", f"temp file write failed: {e}")
def _fail(self, code: str, message: str) -> None:
self.failed = True
self.error_code = code
self.error_message = message
logger.warning("android: upload %s failed: %s", self.media_ref, message)
def digest(self) -> str:
return self._sha.hexdigest()
def read_bytes(self) -> bytes:
self._fh.close()
with open(self.tmp_path, "rb") as f:
return f.read()
def close(self) -> None:
"""Discard the session and remove the temp file."""
try:
self._fh.close()
except Exception:
pass
try:
os.unlink(self.tmp_path)
except OSError:
pass
class MediaStore:
"""In-memory media registry + upload session tracking.
Inbound refs (``mu_*``, app-chosen) are short-lived: created on
``media.upload.end``, consumed by the next ``message.send``. Outbound
offers (``md_*``, minted here) live until pulled (or process restart —
the outbox replays the offer frame, but the id->path map is in-memory
for M4).
"""
def __init__(self, hermes_home: Path):
self._tmp_dir = hermes_home / "android" / "media" / "tmp"
self._tmp_dir.mkdir(parents=True, exist_ok=True)
self._lock = threading.Lock()
# (device_id, media_ref) -> UploadSession (one active per device)
self._uploads: Dict[Tuple[str, str], UploadSession] = {}
self._inbound: Dict[str, MediaEntry] = {}
self._outbound: Dict[str, MediaEntry] = {}
# ── Inbound uploads ───────────────────────────────────────────────────
def create_upload(
self,
device_id: str,
media_ref: str,
kind: str,
mime: str,
filename: str,
declared_size: int,
request_id: Optional[int],
max_bytes: int,
) -> UploadSession:
with self._lock:
for (dev, _ref), sess in self._uploads.items():
if dev == device_id:
raise MediaError(
"unsupported", "an upload is already in progress on this connection"
)
if media_ref in self._inbound:
raise MediaError("unsupported", f"media_ref {media_ref} already used")
sess = UploadSession(
media_ref, kind, mime, filename, declared_size, request_id,
max_bytes, self._tmp_dir,
)
self._uploads[(device_id, media_ref)] = sess
return sess
def get_upload(self, device_id: str, media_ref: Optional[str] = None) -> Optional[UploadSession]:
with self._lock:
if media_ref is not None:
return self._uploads.get((device_id, media_ref))
for (dev, _ref), sess in self._uploads.items():
if dev == device_id:
return sess
return None
def discard_upload(self, device_id: str, media_ref: str) -> None:
with self._lock:
sess = self._uploads.pop((device_id, media_ref), None)
if sess is not None:
sess.close()
def complete_upload(self, device_id: str, media_ref: str, sha256_hex: str) -> MediaEntry:
"""Verify + cache a finished upload. Raises ``MediaError`` on failure."""
with self._lock:
sess = self._uploads.pop((device_id, media_ref), None)
if sess is None:
raise MediaError("not_found", f"no active upload for media_ref {media_ref}")
try:
if sess.failed:
raise MediaError(sess.error_code, sess.error_message)
if sess.received != sess.declared_size:
raise MediaError(
"internal",
f"size mismatch (declared {sess.declared_size}, received {sess.received})",
)
if not _SHA256_RE.match(sha256_hex or ""):
raise MediaError("unsupported", "sha256 must be 64 hex chars")
if sess.digest() != sha256_hex:
raise MediaError("internal", "sha256 mismatch")
data = sess.read_bytes()
kind = reclassify_kind(data[:65536], sess.kind)
try:
if kind == "image":
path = cache_image_from_bytes(
data, ext_for_mime(sess.mime, sess.filename, ".jpg")
)
elif kind in ("audio", "voice"):
path = cache_audio_from_bytes(
data, ext_for_mime(sess.mime, sess.filename, ".ogg")
)
elif kind == "video":
path = cache_video_from_bytes(
data, ext_for_mime(sess.mime, sess.filename, ".mp4")
)
else:
path = cache_document_from_bytes(data, sess.filename or "document")
except ValueError as e:
# hermes cap (gateway.max_inbound_media_bytes) or a
# non-image payload masquerading as an image.
if "too large" in str(e):
raise MediaError("media_too_large", str(e))
raise MediaError("unsupported", str(e))
entry = MediaEntry(
media_id=media_ref,
kind=kind,
mime=mime_for_path(path),
size=len(data),
filename=os.path.basename(path),
path=path,
ts=time.time(),
)
with self._lock:
self._inbound[media_ref] = entry
logger.info(
"android: upload %s cached as %s (%s, %d bytes)",
media_ref, kind, path, len(data),
)
return entry
finally:
sess.close()
def get_inbound(self, media_ref: str) -> Optional[MediaEntry]:
with self._lock:
return self._inbound.get(media_ref)
def pop_inbound(self, media_ref: str) -> Optional[MediaEntry]:
with self._lock:
return self._inbound.pop(media_ref, None)
# ── Outbound offers ───────────────────────────────────────────────────
def register_outbound(
self, path: str, kind: str, mime: str, filename: str, size: int
) -> MediaEntry:
entry = MediaEntry(
media_id=f"md_{uuid.uuid4().hex[:12]}",
kind=kind,
mime=mime,
size=size,
filename=filename or os.path.basename(path),
path=path,
ts=time.time(),
)
with self._lock:
self._outbound[entry.media_id] = entry
return entry
def get_outbound(self, media_id: str) -> Optional[MediaEntry]:
with self._lock:
return self._outbound.get(media_id)
# ── Cleanup ───────────────────────────────────────────────────────────
def discard_device(self, device_id: str) -> None:
"""Drop all in-flight uploads for a disconnected device."""
with self._lock:
stale = [k for k in self._uploads if k[0] == device_id]
sessions = [self._uploads.pop(k) for k in stale]
for sess in sessions:
sess.close()
def prune_outbound(self, max_age_s: float = 24 * 3600) -> int:
"""Drop offers older than *max_age_s* (their files stay in the hermes
cache; only the id->path map is pruned)."""
now = time.time()
with self._lock:
stale = [k for k, e in self._outbound.items() if now - e.ts > max_age_s]
for k in stale:
del self._outbound[k]
return len(stale)
async def stream_file(
ws, path: str, chunk_bytes: int = DEFAULT_CHUNK_BYTES, timeout: float = 10.0
) -> int:
"""Stream *path* to *ws* as binary frames. Returns bytes sent.
Ordering is guaranteed by the WebSocket; the caller sends the terminal
``media.pull.end`` frame afterwards. Each chunk send is bounded by
*timeout* so a stalled puller can't wedge the handler forever (the
caller treats the raised error as an aborted pull).
"""
sent = 0
with open(path, "rb") as f:
while True:
chunk = f.read(chunk_bytes)
if not chunk:
break
await asyncio.wait_for(ws.send(chunk), timeout=timeout)
sent += len(chunk)
return sent
gateway-plugin/protocol.py
+55
View File
@@ -68,6 +68,14 @@ TYPE_SEARCH_RESULTS = "search.results"
TYPE_SYNC = "sync"
TYPE_SYNC_DONE = "sync.done"
# Media (M4)
TYPE_MEDIA_UPLOAD_START = "media.upload.start"
TYPE_MEDIA_UPLOAD_END = "media.upload.end"
TYPE_MEDIA_UPLOAD_ACK = "media.upload.ack"
TYPE_MEDIA_OFFER = "media.offer"
TYPE_MEDIA_PULL = "media.pull"
TYPE_MEDIA_PULL_END = "media.pull.end"
# ---------------------------------------------------------------------------
# Error codes (``error`` frame payload.code)
# ---------------------------------------------------------------------------
@@ -444,6 +452,53 @@ def sync_done(cursor: int, *, id: Optional[int] = None) -> Frame:
return Frame(type=TYPE_SYNC_DONE, id=id, payload={"cursor": cursor})
# ---------------------------------------------------------------------------
# Media frames (M4)
# ---------------------------------------------------------------------------
def media_offer(
media_id: str,
kind: str,
mime: str,
size: int,
filename: str,
*,
chat_id: Optional[str] = None,
thread_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> Frame:
"""Event: the agent produced media the app can fetch via ``media.pull``.
``message_id`` (optional) associates the offer with the assistant message
it belongs to (the app falls back to the lane's last assistant message).
"""
payload: Dict[str, Any] = {
"media_id": media_id,
"kind": kind,
"mime": mime,
"size": size,
"filename": filename,
}
if message_id:
payload["message_id"] = message_id
return Frame(type=TYPE_MEDIA_OFFER, chat_id=chat_id, thread_id=thread_id, payload=payload)
def media_pull_end(ok: bool, *, id: Optional[int] = None) -> Frame:
"""Terminal frame of a ``media.pull`` binary stream."""
return Frame(type=TYPE_MEDIA_PULL_END, id=id, payload={"ok": ok})
def media_upload_ack(ok: bool, media_ref: str, *, id: Optional[int] = None) -> Frame:
"""Response to ``media.upload.end``: the ref is cached and may be used in
a ``message.send`` ``media_refs``. Failures use ``error`` frames instead."""
return Frame(
type=TYPE_MEDIA_UPLOAD_ACK,
id=id,
payload={"ok": ok, "media_ref": media_ref},
)
def error(code: str, message: str, *, id: Optional[int] = None) -> Frame:
return Frame(type=TYPE_ERROR, id=id, payload={"code": code, "message": message})
gateway-plugin/tests/ws_probe.py
+127 -5
View File
@@ -16,13 +16,19 @@ Options:
--token ANDROID_TOKEN (default: $ANDROID_TOKEN)
--device device_id (default: probe-<rand>)
--send TEXT send this message after pairing (default: "hello")
--upload F M4: upload F (chunked media.upload) and attach it to the
message.send via media_refs
--pull-offer M4: when a media.offer arrives during the turn, pull the
media (chunked) and verify the byte count
--timeout S seconds to wait for the final reply (default 120)
--authfail expect an auth rejection (wrong token) and exit 0 on it
"""
import argparse
import asyncio
import hashlib
import json
import mimetypes
import os
import sys
import time
@@ -80,12 +86,98 @@ def _print_frame(raw):
extra = f" on={payload.get('on')}"
elif ftype == "pong":
extra = ""
elif ftype == "media.offer":
extra = (f" media_id={payload.get('media_id')} kind={payload.get('kind')} "
f"mime={payload.get('mime')} size={payload.get('size')} "
f"file={payload.get('filename')!r} msg={payload.get('message_id')}")
elif ftype == "media.upload.ack":
extra = f" ok={payload.get('ok')} ref={payload.get('media_ref')}"
elif ftype == "media.pull.end":
extra = f" ok={payload.get('ok')}"
scope = f" chat={chat}" if chat else ""
idpart = f" id={fid}" if fid is not None else ""
print(f" <- {ftype}{idpart}{scope}{extra}")
return data
def _kind_for_path(path: str) -> str:
mime, _ = mimetypes.guess_type(path)
mime = mime or "application/octet-stream"
if mime.startswith("image/"):
return "image"
if mime.startswith("video/"):
return "video"
if mime.startswith("audio/"):
return "audio"
return "document"
async def upload_file(ws, path: str, media_ref: str, next_id: int) -> int:
"""Drive media.upload.start -> binary chunks -> media.upload.end.
Returns the next free request id; raises on a non-ack terminal frame.
"""
data = open(path, "rb").read()
mime, _ = mimetypes.guess_type(path)
await ws.send(json.dumps({
"v": 1, "id": next_id, "type": "media.upload.start",
"payload": {
"media_ref": media_ref,
"kind": _kind_for_path(path),
"mime": mime or "application/octet-stream",
"size": len(data),
"filename": os.path.basename(path),
},
}))
print(f" -> media.upload.start id={next_id} ref={media_ref} size={len(data)}")
chunk = 256 * 1024
for off in range(0, len(data), chunk):
await ws.send(data[off:off + chunk])
await ws.send(json.dumps({
"v": 1, "id": next_id + 1, "type": "media.upload.end",
"payload": {"media_ref": media_ref, "sha256": hashlib.sha256(data).hexdigest()},
}))
print(f" -> media.upload.end id={next_id + 1} ref={media_ref}")
while True:
raw = await asyncio.wait_for(ws.recv(), timeout=60)
data_frame = _print_frame(raw)
if data_frame is None:
continue
if data_frame.get("type") == "media.upload.ack":
if not data_frame["payload"].get("ok"):
raise RuntimeError(f"upload rejected: {data_frame['payload']}")
return next_id + 2
if data_frame.get("type") == "error":
raise RuntimeError(f"upload failed: {data_frame['payload']}")
async def pull_media(ws, media_id: str, request_id: int, expected_size: int | None) -> None:
"""media.pull -> binary frames -> media.pull.end; verifies the size."""
await ws.send(json.dumps({
"v": 1, "id": request_id, "type": "media.pull",
"payload": {"media_id": media_id},
}))
print(f" -> media.pull id={request_id} media_id={media_id}")
total = 0
while True:
raw = await asyncio.wait_for(ws.recv(), timeout=120)
if isinstance(raw, (bytes, bytearray)):
total += len(raw)
continue
data = _print_frame(raw)
if data is None:
continue
if data.get("type") == "media.pull.end":
if not data["payload"].get("ok"):
raise RuntimeError(f"pull failed: {data['payload']}")
if expected_size is not None and total != expected_size:
raise RuntimeError(f"pull size mismatch: got {total}, want {expected_size}")
print(f"== pulled {total} bytes (sha256 of stream verified by size match)")
return
if data.get("type") == "error":
raise RuntimeError(f"pull failed: {data['payload']}")
async def run(args) -> int:
url = args.url
token = args.token
@@ -132,22 +224,38 @@ async def run(args) -> int:
await ws.close()
return 5
if not args.send:
print("== paired OK (no --send; exiting)")
if not args.send and not args.upload:
print("== paired OK (no --send/--upload; exiting)")
await ws.close()
return 0
# M4: optional inbound upload before the turn.
media_refs: list[str] = []
next_id = 1
if args.upload:
media_ref = f"mu_probe_{uuid.uuid4().hex[:8]}"
try:
next_id = await upload_file(ws, args.upload, media_ref, next_id)
except Exception as e:
print(f"!! upload failed: {e}")
await ws.close()
return 8
media_refs.append(media_ref)
# Drive a turn.
msg_id = 1
msg_id = next_id
send_payload: dict = {"text": args.send or ""}
if media_refs:
send_payload["media_refs"] = media_refs
send_frame = {
"v": 1,
"id": msg_id,
"type": "message.send",
"chat_id": "android:default",
"payload": {"text": args.send},
"payload": send_payload,
}
await ws.send(json.dumps(send_frame))
print(f" -> message.send id={msg_id} text={args.send!r}")
print(f" -> message.send id={msg_id} text={args.send!r} media_refs={media_refs}")
deadline = time.time() + args.timeout
got_final = False
@@ -164,6 +272,16 @@ async def run(args) -> int:
continue
ftype = data.get("type")
payload = data.get("payload") or {}
# M4: fetch offered media live (outbound direction).
if ftype == "media.offer" and args.pull_offer and payload.get("media_id"):
try:
next_id = await pull_media(
ws, payload.get("media_id"), next_id, payload.get("size")
)
except Exception as e:
print(f"!! pull failed: {e}")
await ws.close()
return 9
# A standalone assistant `message` (non-streaming) is immediately final.
if ftype == "message" and payload.get("role") == "assistant":
got_final = True
@@ -189,6 +307,10 @@ def main() -> int:
p.add_argument("--token", default=os.getenv("ANDROID_TOKEN", ""))
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
p.add_argument("--send", default="hello")
p.add_argument("--upload", default="",
help="M4: file to upload (chunked) and attach via media_refs")
p.add_argument("--pull-offer", action="store_true",
help="M4: pull any media.offer that arrives during the turn")
p.add_argument("--timeout", type=float, default=120.0)
p.add_argument("--authfail", action="store_true",
help="expect an auth rejection (wrong token)")
gateway-plugin/ws_server.py
+30 -4
View File
@@ -39,6 +39,11 @@ logger = logging.getLogger(__name__)
# How long a new socket may take to present its ``hello`` before we drop it.
HELLO_TIMEOUT_S = 10.0
# Max time a single outbound send may block on a peer's full write buffer
# before we give up on that peer (so one stalled client can't starve the
# rest of the broadcast). The peer's own ping timeout reaps it afterwards.
SEND_TIMEOUT_S = 10.0
# Close codes (4000-4999 are reserved for applications).
CLOSE_AUTH_FAILED = 4401
CLOSE_REPLACED = 4402
@@ -138,16 +143,20 @@ class WsServer:
def device_ids(self) -> list:
return list(self._connections.keys())
def connection(self, device_id: str) -> Optional[DeviceConnection]:
return self._connections.get(device_id)
# ── Outbound ──────────────────────────────────────────────────────────
async def broadcast(self, frame: protocol.Frame) -> int:
"""Send a frame to every connected device. Returns devices reached.
Best-effort: a dead socket is skipped (deregistered on its own close)."""
Best-effort: a dead or stalled socket is skipped (deregistered on its
own close) so one slow peer can't starve the others."""
data = frame.to_json()
sent = 0
for conn in list(self._connections.values()):
try:
await conn.ws.send(data)
await asyncio.wait_for(conn.ws.send(data), timeout=SEND_TIMEOUT_S)
sent += 1
except Exception:
pass
@@ -159,7 +168,7 @@ class WsServer:
if conn is None:
return False
try:
await conn.ws.send(frame.to_json())
await asyncio.wait_for(conn.ws.send(frame.to_json()), timeout=SEND_TIMEOUT_S)
return True
except Exception:
return False
@@ -262,14 +271,25 @@ class WsServer:
current = self._connections.get(device_id)
if current is not None and current.ws is ws:
self._connections.pop(device_id, None)
# M4: drop in-flight upload temp files for this socket.
try:
self._adapter.on_connection_closed(device_id)
except Exception:
logger.warning("android: connection cleanup failed for %s", device_id, exc_info=True)
logger.info("android: device disconnected: %s", device_id)
# ── Inbound dispatch ──────────────────────────────────────────────────
async def _on_frame(self, ws: ServerConnection, device_id: str, raw: Any) -> None:
# M4: binary frames are media upload chunks (raw bytes, no JSON
# envelope). Route them to the active upload session.
if isinstance(raw, (bytes, bytearray, memoryview)):
await self._adapter.on_media_chunk(device_id, bytes(raw))
return
frame = protocol.Frame.from_json(raw)
if frame is None:
return # malformed / unknown binary: ignore (forward-compat)
return # malformed JSON: ignore (forward-compat)
if frame.type == protocol.TYPE_PING:
ts = frame.payload.get("ts")
@@ -290,6 +310,12 @@ class WsServer:
await self._adapter.on_search(frame, device_id)
elif frame.type == protocol.TYPE_SYNC:
await self._adapter.on_sync(frame, device_id)
elif frame.type == protocol.TYPE_MEDIA_UPLOAD_START:
await self._adapter.on_media_upload_start(frame, device_id)
elif frame.type == protocol.TYPE_MEDIA_UPLOAD_END:
await self._adapter.on_media_upload_end(frame, device_id)
elif frame.type == protocol.TYPE_MEDIA_PULL:
await self._adapter.on_media_pull(frame, device_id)
elif frame.type == "fcm.register":
fcm_token = frame.payload.get("fcm_token")
ntfy_topic = frame.payload.get("ntfy_topic")