ARIA/iris_x_hermes
1
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases 4 Wiki Activity

M4: media upload/download/playback (both directions)

Browse Source 
Gateway plugin:
- media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff ->
  cache_*_from_bytes -> media.upload.ack
- media.offer / media.pull (chunked) for agent-sent media, delivery-path
  security re-checked at pull time
- send_* overrides mint media_id and emit media.offer
- message.send media_refs resolve to cached inbound media
- per-send + per-chunk timeouts so a stalled peer can't starve the rest

App (Kotlin CMP):
- Protocol: media frame types/payloads/builders
- GatewayClient: binary session, uploadMedia (chunked + streaming sha256),
  pullMedia serialized via Mutex so concurrent offers don't interleave
- ChatStore/IrisController: MediaItem, attachments, auto-pull on offer
- Platform media: SAF picker, ExoPlayer (audio mini-player + video), image
  loader, FileProvider document open (Android); AWT-free desktop actuals
- ChatScreen: attach button + chips, media rendering, keyboard dismiss on send

UI polish:
- preserve image aspect ratio (no stretching), cap dominant dimension
- adjustResize so only chat content squeezes for the keyboard
- clear focus (hide keyboard) on send

Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json +
07-media.md; M4 marked complete in 14-milestones.md.

Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
ARIA committed 2026-08-19 17:29:39 +02:00
1 parent 60296b33fe
commit 913ee91024
28 files changed
+2234 -53
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

gateway-plugin/tests/ws_probe.py
+127 -5
View File
@@ -16,13 +16,19 @@ Options:
--token ANDROID_TOKEN (default: $ANDROID_TOKEN)
--device device_id (default: probe-<rand>)
--send TEXT send this message after pairing (default: "hello")
--upload F M4: upload F (chunked media.upload) and attach it to the
message.send via media_refs
--pull-offer M4: when a media.offer arrives during the turn, pull the
media (chunked) and verify the byte count
--timeout S seconds to wait for the final reply (default 120)
--authfail expect an auth rejection (wrong token) and exit 0 on it
"""
import argparse
import asyncio
import hashlib
import json
import mimetypes
import os
import sys
import time
@@ -80,12 +86,98 @@ def _print_frame(raw):
extra = f" on={payload.get('on')}"
elif ftype == "pong":
extra = ""
elif ftype == "media.offer":
extra = (f" media_id={payload.get('media_id')} kind={payload.get('kind')} "
f"mime={payload.get('mime')} size={payload.get('size')} "
f"file={payload.get('filename')!r} msg={payload.get('message_id')}")
elif ftype == "media.upload.ack":
extra = f" ok={payload.get('ok')} ref={payload.get('media_ref')}"
elif ftype == "media.pull.end":
extra = f" ok={payload.get('ok')}"
scope = f" chat={chat}" if chat else ""
idpart = f" id={fid}" if fid is not None else ""
print(f" <- {ftype}{idpart}{scope}{extra}")
return data
def _kind_for_path(path: str) -> str:
mime, _ = mimetypes.guess_type(path)
mime = mime or "application/octet-stream"
if mime.startswith("image/"):
return "image"
if mime.startswith("video/"):
return "video"
if mime.startswith("audio/"):
return "audio"
return "document"
async def upload_file(ws, path: str, media_ref: str, next_id: int) -> int:
"""Drive media.upload.start -> binary chunks -> media.upload.end.
Returns the next free request id; raises on a non-ack terminal frame.
"""
data = open(path, "rb").read()
mime, _ = mimetypes.guess_type(path)
await ws.send(json.dumps({
"v": 1, "id": next_id, "type": "media.upload.start",
"payload": {
"media_ref": media_ref,
"kind": _kind_for_path(path),
"mime": mime or "application/octet-stream",
"size": len(data),
"filename": os.path.basename(path),
},
}))
print(f" -> media.upload.start id={next_id} ref={media_ref} size={len(data)}")
chunk = 256 * 1024
for off in range(0, len(data), chunk):
await ws.send(data[off:off + chunk])
await ws.send(json.dumps({
"v": 1, "id": next_id + 1, "type": "media.upload.end",
"payload": {"media_ref": media_ref, "sha256": hashlib.sha256(data).hexdigest()},
}))
print(f" -> media.upload.end id={next_id + 1} ref={media_ref}")
while True:
raw = await asyncio.wait_for(ws.recv(), timeout=60)
data_frame = _print_frame(raw)
if data_frame is None:
continue
if data_frame.get("type") == "media.upload.ack":
if not data_frame["payload"].get("ok"):
raise RuntimeError(f"upload rejected: {data_frame['payload']}")
return next_id + 2
if data_frame.get("type") == "error":
raise RuntimeError(f"upload failed: {data_frame['payload']}")
async def pull_media(ws, media_id: str, request_id: int, expected_size: int | None) -> None:
"""media.pull -> binary frames -> media.pull.end; verifies the size."""
await ws.send(json.dumps({
"v": 1, "id": request_id, "type": "media.pull",
"payload": {"media_id": media_id},
}))
print(f" -> media.pull id={request_id} media_id={media_id}")
total = 0
while True:
raw = await asyncio.wait_for(ws.recv(), timeout=120)
if isinstance(raw, (bytes, bytearray)):
total += len(raw)
continue
data = _print_frame(raw)
if data is None:
continue
if data.get("type") == "media.pull.end":
if not data["payload"].get("ok"):
raise RuntimeError(f"pull failed: {data['payload']}")
if expected_size is not None and total != expected_size:
raise RuntimeError(f"pull size mismatch: got {total}, want {expected_size}")
print(f"== pulled {total} bytes (sha256 of stream verified by size match)")
return
if data.get("type") == "error":
raise RuntimeError(f"pull failed: {data['payload']}")
async def run(args) -> int:
url = args.url
token = args.token
@@ -132,22 +224,38 @@ async def run(args) -> int:
await ws.close()
return 5
if not args.send:
print("== paired OK (no --send; exiting)")
if not args.send and not args.upload:
print("== paired OK (no --send/--upload; exiting)")
await ws.close()
return 0
# M4: optional inbound upload before the turn.
media_refs: list[str] = []
next_id = 1
if args.upload:
media_ref = f"mu_probe_{uuid.uuid4().hex[:8]}"
try:
next_id = await upload_file(ws, args.upload, media_ref, next_id)
except Exception as e:
print(f"!! upload failed: {e}")
await ws.close()
return 8
media_refs.append(media_ref)
# Drive a turn.
msg_id = 1
msg_id = next_id
send_payload: dict = {"text": args.send or ""}
if media_refs:
send_payload["media_refs"] = media_refs
send_frame = {
"v": 1,
"id": msg_id,
"type": "message.send",
"chat_id": "android:default",
"payload": {"text": args.send},
"payload": send_payload,
}
await ws.send(json.dumps(send_frame))
print(f" -> message.send id={msg_id} text={args.send!r}")
print(f" -> message.send id={msg_id} text={args.send!r} media_refs={media_refs}")
deadline = time.time() + args.timeout
got_final = False
@@ -164,6 +272,16 @@ async def run(args) -> int:
continue
ftype = data.get("type")
payload = data.get("payload") or {}
# M4: fetch offered media live (outbound direction).
if ftype == "media.offer" and args.pull_offer and payload.get("media_id"):
try:
next_id = await pull_media(
ws, payload.get("media_id"), next_id, payload.get("size")
)
except Exception as e:
print(f"!! pull failed: {e}")
await ws.close()
return 9
# A standalone assistant `message` (non-streaming) is immediately final.
if ftype == "message" and payload.get("role") == "assistant":
got_final = True
@@ -189,6 +307,10 @@ def main() -> int:
p.add_argument("--token", default=os.getenv("ANDROID_TOKEN", ""))
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
p.add_argument("--send", default="hello")
p.add_argument("--upload", default="",
help="M4: file to upload (chunked) and attach via media_refs")
p.add_argument("--pull-offer", action="store_true",
help="M4: pull any media.offer that arrives during the turn")
p.add_argument("--timeout", type=float, default=120.0)
p.add_argument("--authfail", action="store_true",
help="expect an auth rejection (wrong token)")