M4: media upload/download/playback (both directions)
Gateway plugin: - media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff -> cache_*_from_bytes -> media.upload.ack - media.offer / media.pull (chunked) for agent-sent media, delivery-path security re-checked at pull time - send_* overrides mint media_id and emit media.offer - message.send media_refs resolve to cached inbound media - per-send + per-chunk timeouts so a stalled peer can't starve the rest App (Kotlin CMP): - Protocol: media frame types/payloads/builders - GatewayClient: binary session, uploadMedia (chunked + streaming sha256), pullMedia serialized via Mutex so concurrent offers don't interleave - ChatStore/IrisController: MediaItem, attachments, auto-pull on offer - Platform media: SAF picker, ExoPlayer (audio mini-player + video), image loader, FileProvider document open (Android); AWT-free desktop actuals - ChatScreen: attach button + chips, media rendering, keyboard dismiss on send UI polish: - preserve image aspect ratio (no stretching), cap dominant dimension - adjustResize so only chat content squeezes for the keyboard - clear focus (hide keyboard) on send Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json + 07-media.md; M4 marked complete in 14-milestones.md. Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
1 parent
60296b33fe
commit
913ee91024
28 files changed
+2234
-53
No files matched your search
@@ -68,6 +68,14 @@ TYPE_SEARCH_RESULTS = "search.results"
|
||||
TYPE_SYNC = "sync"
|
||||
TYPE_SYNC_DONE = "sync.done"
|
||||
|
||||
# Media (M4)
|
||||
TYPE_MEDIA_UPLOAD_START = "media.upload.start"
|
||||
TYPE_MEDIA_UPLOAD_END = "media.upload.end"
|
||||
TYPE_MEDIA_UPLOAD_ACK = "media.upload.ack"
|
||||
TYPE_MEDIA_OFFER = "media.offer"
|
||||
TYPE_MEDIA_PULL = "media.pull"
|
||||
TYPE_MEDIA_PULL_END = "media.pull.end"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error codes (``error`` frame payload.code)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -444,6 +452,53 @@ def sync_done(cursor: int, *, id: Optional[int] = None) -> Frame:
|
||||
return Frame(type=TYPE_SYNC_DONE, id=id, payload={"cursor": cursor})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Media frames (M4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def media_offer(
|
||||
media_id: str,
|
||||
kind: str,
|
||||
mime: str,
|
||||
size: int,
|
||||
filename: str,
|
||||
*,
|
||||
chat_id: Optional[str] = None,
|
||||
thread_id: Optional[str] = None,
|
||||
message_id: Optional[str] = None,
|
||||
) -> Frame:
|
||||
"""Event: the agent produced media the app can fetch via ``media.pull``.
|
||||
|
||||
``message_id`` (optional) associates the offer with the assistant message
|
||||
it belongs to (the app falls back to the lane's last assistant message).
|
||||
"""
|
||||
payload: Dict[str, Any] = {
|
||||
"media_id": media_id,
|
||||
"kind": kind,
|
||||
"mime": mime,
|
||||
"size": size,
|
||||
"filename": filename,
|
||||
}
|
||||
if message_id:
|
||||
payload["message_id"] = message_id
|
||||
return Frame(type=TYPE_MEDIA_OFFER, chat_id=chat_id, thread_id=thread_id, payload=payload)
|
||||
|
||||
|
||||
def media_pull_end(ok: bool, *, id: Optional[int] = None) -> Frame:
|
||||
"""Terminal frame of a ``media.pull`` binary stream."""
|
||||
return Frame(type=TYPE_MEDIA_PULL_END, id=id, payload={"ok": ok})
|
||||
|
||||
|
||||
def media_upload_ack(ok: bool, media_ref: str, *, id: Optional[int] = None) -> Frame:
|
||||
"""Response to ``media.upload.end``: the ref is cached and may be used in
|
||||
a ``message.send`` ``media_refs``. Failures use ``error`` frames instead."""
|
||||
return Frame(
|
||||
type=TYPE_MEDIA_UPLOAD_ACK,
|
||||
id=id,
|
||||
payload={"ok": ok, "media_ref": media_ref},
|
||||
)
|
||||
|
||||
|
||||
def error(code: str, message: str, *, id: Optional[int] = None) -> Frame:
|
||||
return Frame(type=TYPE_ERROR, id=id, payload={"code": code, "message": message})
|
||||
|
||||
|
||||
Reference in new issue
Block a user