M4: media upload/download/playback (both directions)
Gateway plugin: - media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff -> cache_*_from_bytes -> media.upload.ack - media.offer / media.pull (chunked) for agent-sent media, delivery-path security re-checked at pull time - send_* overrides mint media_id and emit media.offer - message.send media_refs resolve to cached inbound media - per-send + per-chunk timeouts so a stalled peer can't starve the rest App (Kotlin CMP): - Protocol: media frame types/payloads/builders - GatewayClient: binary session, uploadMedia (chunked + streaming sha256), pullMedia serialized via Mutex so concurrent offers don't interleave - ChatStore/IrisController: MediaItem, attachments, auto-pull on offer - Platform media: SAF picker, ExoPlayer (audio mini-player + video), image loader, FileProvider document open (Android); AWT-free desktop actuals - ChatScreen: attach button + chips, media rendering, keyboard dismiss on send UI polish: - preserve image aspect ratio (no stretching), cap dominant dimension - adjustResize so only chat content squeezes for the keyboard - clear focus (hide keyboard) on send Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json + 07-media.md; M4 marked complete in 14-milestones.md. Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
1 parent
60296b33fe
commit
913ee91024
28 files changed
+2234
-53
No files matched your search
@@ -2,8 +2,14 @@ package iris.state
|
||||
|
||||
import iris.data.ChatStore
|
||||
import iris.data.ChannelStore
|
||||
import iris.data.MediaItem
|
||||
import iris.data.SecureStore
|
||||
import iris.media.MediaCache
|
||||
import iris.media.kindFromMime
|
||||
import iris.net.GatewayClient
|
||||
import iris.platform.PickedFile
|
||||
import iris.platform.mediaCacheBaseDir
|
||||
import iris.protocol.MediaOfferPayload
|
||||
import iris.protocol.SearchHit
|
||||
import iris.protocol.SearchResultsPayload
|
||||
import iris.protocol.SyncDonePayload
|
||||
@@ -12,6 +18,7 @@ import iris.protocol.TYPE_CHANNEL_DELETED
|
||||
import iris.protocol.TYPE_CHANNEL_LIST
|
||||
import iris.protocol.TYPE_CHANNEL_RENAMED
|
||||
import iris.protocol.TYPE_COMMENTARY
|
||||
import iris.protocol.TYPE_MEDIA_OFFER
|
||||
import iris.protocol.TYPE_MESSAGE
|
||||
import iris.protocol.TYPE_MESSAGE_START
|
||||
import iris.protocol.TYPE_MESSAGE_STOP
|
||||
@@ -37,6 +44,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* App-level controller (M3): owns the GatewayClient + ChatStore + ChannelStore,
|
||||
@@ -88,6 +96,24 @@ class IrisController(
|
||||
private val _lastQuery = MutableStateFlow("")
|
||||
val lastQuery: StateFlow<String> = _lastQuery.asStateFlow()
|
||||
|
||||
// ── M4: media ─────────────────────────────────────────────────────────
|
||||
private val mediaCache = MediaCache(mediaCacheBaseDir())
|
||||
|
||||
/** A composer attachment: picked file being uploaded (or uploaded). */
|
||||
data class PendingAttachment(
|
||||
val filename: String,
|
||||
val mime: String,
|
||||
val size: Long,
|
||||
val kind: String,
|
||||
val path: String? = null,
|
||||
val uploading: Boolean = true,
|
||||
val mediaRef: String? = null,
|
||||
val error: String? = null,
|
||||
)
|
||||
|
||||
private val _attachments = MutableStateFlow<List<PendingAttachment>>(emptyList())
|
||||
val attachments: StateFlow<List<PendingAttachment>> = _attachments.asStateFlow()
|
||||
|
||||
init {
|
||||
scope.launch {
|
||||
client.events.collect { frame ->
|
||||
@@ -99,7 +125,14 @@ class IrisController(
|
||||
TYPE_TOOL_START,
|
||||
TYPE_TOOL_PROGRESS,
|
||||
TYPE_TOOL_END,
|
||||
TYPE_COMMENTARY -> chat.onFrame(frame)
|
||||
TYPE_COMMENTARY,
|
||||
TYPE_MEDIA_OFFER -> {
|
||||
chat.onFrame(frame)
|
||||
// M4: pull offered media into the local cache.
|
||||
if (frame.type == TYPE_MEDIA_OFFER) {
|
||||
frame.payloadAs<MediaOfferPayload>()?.let { pullMedia(it) }
|
||||
}
|
||||
}
|
||||
TYPE_CHANNEL_CREATED,
|
||||
TYPE_CHANNEL_RENAMED,
|
||||
TYPE_CHANNEL_DELETED,
|
||||
@@ -202,14 +235,76 @@ class IrisController(
|
||||
client.sendFrame(syncFrame(0, cursor))
|
||||
}
|
||||
|
||||
/** Optimistic send: show immediately in the current lane, then hand to the gateway. */
|
||||
fun send(text: String) {
|
||||
/** Optimistic send: show immediately in the current lane, then hand to the gateway.
|
||||
* M4: [attachments] (uploaded) are attached via media_refs. */
|
||||
fun send(text: String, attachments: List<PendingAttachment> = emptyList()) {
|
||||
val trimmed = text.trim()
|
||||
if (trimmed.isEmpty()) return
|
||||
val refs = attachments.mapNotNull { it.mediaRef }
|
||||
if (trimmed.isEmpty() && refs.isEmpty()) return
|
||||
val lane = chat.currentLane.value
|
||||
val (chatId, threadId) = chat.parseLane(lane)
|
||||
chat.addPending(trimmed, lane)
|
||||
client.sendMessage(chatId, trimmed, threadId)
|
||||
val media = attachments.map {
|
||||
MediaItem(
|
||||
mediaId = it.mediaRef ?: it.filename,
|
||||
kind = it.kind, mime = it.mime, size = it.size, filename = it.filename,
|
||||
localPath = it.path,
|
||||
)
|
||||
}
|
||||
chat.addPending(trimmed, lane, media)
|
||||
client.sendMessage(chatId, trimmed, threadId, refs)
|
||||
_attachments.value = emptyList()
|
||||
}
|
||||
|
||||
// ── M4: attachments (pick -> upload -> send) ──────────────────────────
|
||||
|
||||
/** Stage a picked file: upload it, then keep it as a pending attachment. */
|
||||
fun attachFile(picked: PickedFile) {
|
||||
val kind = kindFromMime(picked.mime)
|
||||
val placeholder = PendingAttachment(
|
||||
filename = picked.name, mime = picked.mime, size = picked.size, kind = kind,
|
||||
path = picked.path,
|
||||
)
|
||||
_attachments.value = _attachments.value + placeholder
|
||||
scope.launch {
|
||||
val mediaRef = "up_${Random.nextLong(1_000_000_000L, 9_999_999_999L)}"
|
||||
val result = client.uploadMedia(
|
||||
path = picked.path,
|
||||
mime = picked.mime,
|
||||
kind = kind,
|
||||
filename = picked.name,
|
||||
mediaRef = mediaRef,
|
||||
)
|
||||
_attachments.value = _attachments.value.map {
|
||||
if (it.filename == picked.name && it.uploading) {
|
||||
result.fold(
|
||||
{ ref -> it.copy(uploading = false, mediaRef = ref) },
|
||||
{ e -> it.copy(uploading = false, error = e.message) },
|
||||
)
|
||||
} else it
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun removeAttachment(filename: String) {
|
||||
_attachments.value = _attachments.value.filterNot { it.filename == filename }
|
||||
}
|
||||
|
||||
/** Pull offered media into the local cache and record the path. */
|
||||
private fun pullMedia(offer: MediaOfferPayload) {
|
||||
scope.launch {
|
||||
// Already cached? Skip the pull.
|
||||
mediaCache.path(offer.mediaId, offer.mime)?.let {
|
||||
chat.setMediaLocalPath(offer.mediaId, it)
|
||||
return@launch
|
||||
}
|
||||
val writer = mediaCache.openWriter(offer.mediaId, offer.mime)
|
||||
val result = client.pullMedia(offer.mediaId) { chunk -> writer.write(chunk) }
|
||||
writer.close()
|
||||
result.fold(
|
||||
{ chat.setMediaLocalPath(offer.mediaId, writer.path) },
|
||||
{ mediaCache.remove(offer.mediaId, offer.mime) },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Connect-screen action: real hello test, then save + (re)connect. */
|
||||
|
||||
Reference in new issue
Block a user