M4: media upload/download/playback (both directions)

Gateway plugin:
- media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff ->
  cache_*_from_bytes -> media.upload.ack
- media.offer / media.pull (chunked) for agent-sent media, delivery-path
  security re-checked at pull time
- send_* overrides mint media_id and emit media.offer
- message.send media_refs resolve to cached inbound media
- per-send + per-chunk timeouts so a stalled peer can't starve the rest

App (Kotlin CMP):
- Protocol: media frame types/payloads/builders
- GatewayClient: binary session, uploadMedia (chunked + streaming sha256),
  pullMedia serialized via Mutex so concurrent offers don't interleave
- ChatStore/IrisController: MediaItem, attachments, auto-pull on offer
- Platform media: SAF picker, ExoPlayer (audio mini-player + video), image
  loader, FileProvider document open (Android); AWT-free desktop actuals
- ChatScreen: attach button + chips, media rendering, keyboard dismiss on send

UI polish:
- preserve image aspect ratio (no stretching), cap dominant dimension
- adjustResize so only chat content squeezes for the keyboard
- clear focus (hide keyboard) on send

Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json +
07-media.md; M4 marked complete in 14-milestones.md.

Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
ARIA committed 2026-08-19 17:29:39 +02:00
1 parent 60296b33fe
commit 913ee91024
28 files changed
+2234 -53

No files matched your search

@@ -2,8 +2,14 @@ package iris.state
import iris.data.ChatStore
import iris.data.ChannelStore
import iris.data.MediaItem
import iris.data.SecureStore
import iris.media.MediaCache
import iris.media.kindFromMime
import iris.net.GatewayClient
import iris.platform.PickedFile
import iris.platform.mediaCacheBaseDir
import iris.protocol.MediaOfferPayload
import iris.protocol.SearchHit
import iris.protocol.SearchResultsPayload
import iris.protocol.SyncDonePayload
@@ -12,6 +18,7 @@ import iris.protocol.TYPE_CHANNEL_DELETED
import iris.protocol.TYPE_CHANNEL_LIST
import iris.protocol.TYPE_CHANNEL_RENAMED
import iris.protocol.TYPE_COMMENTARY
import iris.protocol.TYPE_MEDIA_OFFER
import iris.protocol.TYPE_MESSAGE
import iris.protocol.TYPE_MESSAGE_START
import iris.protocol.TYPE_MESSAGE_STOP
@@ -37,6 +44,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlin.random.Random
/**
* App-level controller (M3): owns the GatewayClient + ChatStore + ChannelStore,
@@ -88,6 +96,24 @@ class IrisController(
private val _lastQuery = MutableStateFlow("")
val lastQuery: StateFlow<String> = _lastQuery.asStateFlow()
// ── M4: media ─────────────────────────────────────────────────────────
private val mediaCache = MediaCache(mediaCacheBaseDir())
/** A composer attachment: picked file being uploaded (or uploaded). */
data class PendingAttachment(
val filename: String,
val mime: String,
val size: Long,
val kind: String,
val path: String? = null,
val uploading: Boolean = true,
val mediaRef: String? = null,
val error: String? = null,
)
private val _attachments = MutableStateFlow<List<PendingAttachment>>(emptyList())
val attachments: StateFlow<List<PendingAttachment>> = _attachments.asStateFlow()
init {
scope.launch {
client.events.collect { frame ->
@@ -99,7 +125,14 @@ class IrisController(
TYPE_TOOL_START,
TYPE_TOOL_PROGRESS,
TYPE_TOOL_END,
TYPE_COMMENTARY -> chat.onFrame(frame)
TYPE_COMMENTARY,
TYPE_MEDIA_OFFER -> {
chat.onFrame(frame)
// M4: pull offered media into the local cache.
if (frame.type == TYPE_MEDIA_OFFER) {
frame.payloadAs<MediaOfferPayload>()?.let { pullMedia(it) }
}
}
TYPE_CHANNEL_CREATED,
TYPE_CHANNEL_RENAMED,
TYPE_CHANNEL_DELETED,
@@ -202,14 +235,76 @@ class IrisController(
client.sendFrame(syncFrame(0, cursor))
}
/** Optimistic send: show immediately in the current lane, then hand to the gateway. */
fun send(text: String) {
/** Optimistic send: show immediately in the current lane, then hand to the gateway.
* M4: [attachments] (uploaded) are attached via media_refs. */
fun send(text: String, attachments: List<PendingAttachment> = emptyList()) {
val trimmed = text.trim()
if (trimmed.isEmpty()) return
val refs = attachments.mapNotNull { it.mediaRef }
if (trimmed.isEmpty() && refs.isEmpty()) return
val lane = chat.currentLane.value
val (chatId, threadId) = chat.parseLane(lane)
chat.addPending(trimmed, lane)
client.sendMessage(chatId, trimmed, threadId)
val media = attachments.map {
MediaItem(
mediaId = it.mediaRef ?: it.filename,
kind = it.kind, mime = it.mime, size = it.size, filename = it.filename,
localPath = it.path,
)
}
chat.addPending(trimmed, lane, media)
client.sendMessage(chatId, trimmed, threadId, refs)
_attachments.value = emptyList()
}
// ── M4: attachments (pick -> upload -> send) ──────────────────────────
/** Stage a picked file: upload it, then keep it as a pending attachment. */
fun attachFile(picked: PickedFile) {
val kind = kindFromMime(picked.mime)
val placeholder = PendingAttachment(
filename = picked.name, mime = picked.mime, size = picked.size, kind = kind,
path = picked.path,
)
_attachments.value = _attachments.value + placeholder
scope.launch {
val mediaRef = "up_${Random.nextLong(1_000_000_000L, 9_999_999_999L)}"
val result = client.uploadMedia(
path = picked.path,
mime = picked.mime,
kind = kind,
filename = picked.name,
mediaRef = mediaRef,
)
_attachments.value = _attachments.value.map {
if (it.filename == picked.name && it.uploading) {
result.fold(
{ ref -> it.copy(uploading = false, mediaRef = ref) },
{ e -> it.copy(uploading = false, error = e.message) },
)
} else it
}
}
}
fun removeAttachment(filename: String) {
_attachments.value = _attachments.value.filterNot { it.filename == filename }
}
/** Pull offered media into the local cache and record the path. */
private fun pullMedia(offer: MediaOfferPayload) {
scope.launch {
// Already cached? Skip the pull.
mediaCache.path(offer.mediaId, offer.mime)?.let {
chat.setMediaLocalPath(offer.mediaId, it)
return@launch
}
val writer = mediaCache.openWriter(offer.mediaId, offer.mime)
val result = client.pullMedia(offer.mediaId) { chunk -> writer.write(chunk) }
writer.close()
result.fold(
{ chat.setMediaLocalPath(offer.mediaId, writer.path) },
{ mediaCache.remove(offer.mediaId, offer.mime) },
)
}
}
/** Connect-screen action: real hello test, then save + (re)connect. */