M4: media upload/download/playback (both directions)

Gateway plugin:
- media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff ->
  cache_*_from_bytes -> media.upload.ack
- media.offer / media.pull (chunked) for agent-sent media, delivery-path
  security re-checked at pull time
- send_* overrides mint media_id and emit media.offer
- message.send media_refs resolve to cached inbound media
- per-send + per-chunk timeouts so a stalled peer can't starve the rest

App (Kotlin CMP):
- Protocol: media frame types/payloads/builders
- GatewayClient: binary session, uploadMedia (chunked + streaming sha256),
  pullMedia serialized via Mutex so concurrent offers don't interleave
- ChatStore/IrisController: MediaItem, attachments, auto-pull on offer
- Platform media: SAF picker, ExoPlayer (audio mini-player + video), image
  loader, FileProvider document open (Android); AWT-free desktop actuals
- ChatScreen: attach button + chips, media rendering, keyboard dismiss on send

UI polish:
- preserve image aspect ratio (no stretching), cap dominant dimension
- adjustResize so only chat content squeezes for the keyboard
- clear focus (hide keyboard) on send

Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json +
07-media.md; M4 marked complete in 14-milestones.md.

Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
ARIA committed 2026-08-19 17:29:39 +02:00
1 parent 60296b33fe
commit 913ee91024
28 files changed
+2234 -53

No files matched your search

+13
View File
@@ -16,6 +16,12 @@ kotlin {
jvm("desktop")
sourceSets {
// Both targets are JVM-based (androidTarget + jvm("desktop")), so
// shared JVM code (File I/O, SHA-256, media cache) lives in jvmMain.
val jvmMain by creating { dependsOn(commonMain.get()) }
val androidMain by getting { dependsOn(jvmMain) }
val desktopMain by getting { dependsOn(jvmMain) }
commonMain.dependencies {
implementation("org.jetbrains.compose.runtime:runtime:$composeVersion")
implementation("org.jetbrains.compose.foundation:foundation:$composeVersion")
@@ -27,6 +33,13 @@ kotlin {
// (androidTarget + jvm("desktop")), so it resolves on both.
implementation("com.squareup.okhttp3:okhttp:$okhttpVersion")
}
// M4: ExoPlayer (Media3) for inline audio/video playback (Android only).
androidMain.dependencies {
implementation("androidx.media3:media3-exoplayer:1.3.1")
implementation("androidx.media3:media3-ui:1.3.1")
// SAF picker (rememberLauncherForActivityResult).
implementation("androidx.activity:activity-compose:1.9.2")
}
}
}
@@ -0,0 +1,218 @@
package iris.platform
import android.content.Context
import android.content.Intent
import android.graphics.BitmapFactory
import android.net.Uri
import android.provider.OpenableColumns
import android.widget.Toast
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.material3.Button
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Slider
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.core.content.FileProvider
import androidx.core.net.toUri
import androidx.media3.common.MediaItem as M3MediaItem
import androidx.media3.common.Player
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.ui.PlayerView
import iris.data.MediaItem
import iris.media.extForMime
import iris.protocol.KIND_VIDEO
import java.io.File
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
/** App context holder (set in MainActivity.onCreate). */
object AndroidEnv {
lateinit var context: Context
}
@Composable
actual fun MediaFilePicker(onPicked: (PickedFile?) -> Unit) {
val context = LocalContext.current
val launcher = rememberLauncherForActivityResult(ActivityResultContracts.GetContent()) { uri ->
if (uri == null) {
onPicked(null)
return@rememberLauncherForActivityResult
}
try {
val mime = context.contentResolver.getType(uri) ?: "application/octet-stream"
val name = queryDisplayName(context, uri) ?: "upload"
val dir = File(context.cacheDir, "uploads").apply { mkdirs() }
val target = File(dir, "${System.currentTimeMillis()}${extForMime(mime)}")
context.contentResolver.openInputStream(uri)!!.use { input ->
target.outputStream().use { output -> input.copyTo(output) }
}
onPicked(PickedFile(target.absolutePath, name, mime, target.length()))
} catch (e: Exception) {
onPicked(null)
}
}
Box(modifier = Modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
Button(onClick = { launcher.launch("*/*") }) { Text("Pick a file") }
}
}
private fun queryDisplayName(context: Context, uri: Uri): String? {
context.contentResolver.query(uri, null, null, null, null)?.use { c ->
val idx = c.getColumnIndex(OpenableColumns.DISPLAY_NAME)
if (idx >= 0 && c.moveToFirst()) return c.getString(idx)
}
return null
}
actual fun mediaCacheBaseDir(): String =
AndroidEnv.context.externalCacheDir?.absolutePath
?: AndroidEnv.context.cacheDir.absolutePath
actual suspend fun loadMediaImage(path: String): ImageBitmap? = withContext(Dispatchers.IO) {
try {
BitmapFactory.decodeFile(path)?.asImageBitmap()
} catch (e: Exception) {
null
}
}
@Composable
actual fun MediaPlayerView(media: MediaItem, modifier: Modifier) {
val path = media.localPath
if (path == null) {
Box(modifier = modifier, contentAlignment = Alignment.Center) {
Row {
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
Spacer(modifier = Modifier.size(8.dp))
Text("downloading…", fontSize = 12.sp)
}
}
return
}
if (media.kind == KIND_VIDEO) {
VideoPlayerView(path, modifier)
} else {
AudioPlayerView(path, media.filename, modifier)
}
}
@Composable
private fun VideoPlayerView(path: String, modifier: Modifier) {
AndroidView(
factory = { ctx ->
PlayerView(ctx).apply {
useController = true
val player = ExoPlayer.Builder(ctx).build()
player.setMediaItem(M3MediaItem.fromUri(path.toUri()))
player.prepare()
player.playWhenReady = true
this.player = player
}
},
update = {},
onRelease = { view -> view.player?.release() },
modifier = modifier
.fillMaxWidth()
.height(220.dp),
)
}
@Composable
private fun AudioPlayerView(path: String, filename: String, modifier: Modifier) {
val context = LocalContext.current
val player = remember {
ExoPlayer.Builder(context).build().apply {
setMediaItem(M3MediaItem.fromUri(path.toUri()))
prepare()
}
}
DisposableEffect(player) { onDispose { player.release() } }
var playing by remember { mutableStateOf(false) }
var position by remember { mutableFloatStateOf(0f) }
var duration by remember { mutableFloatStateOf(0f) }
LaunchedEffect(player) {
while (true) {
val d = player.duration
duration = if (d > 0) d / 1000f else 0f
position = player.currentPosition / 1000f
playing = player.isPlaying
kotlinx.coroutines.delay(500)
}
}
Row(
modifier = modifier
.fillMaxWidth()
.padding(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
IconButton(onClick = {
if (playing) player.pause() else player.play()
}) {
Text(if (playing) "⏸" else "▶️", fontSize = 18.sp)
}
Column(modifier = Modifier.weight(1f).padding(horizontal = 8.dp)) {
Text(filename, fontSize = 12.sp, maxLines = 1)
Slider(
value = if (duration > 0f) (position / duration).coerceIn(0f, 1f) else 0f,
onValueChange = { frac ->
player.seekTo((frac * duration * 1000).toLong())
},
modifier = Modifier.fillMaxWidth(),
)
Text(
"${fmt(position)} / ${fmt(duration)}",
fontSize = 10.sp,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
}
private fun fmt(seconds: Float): String {
val s = seconds.toLong()
return "%d:%02d".format(s / 60, s % 60)
}
actual fun openDocument(path: String, mime: String) {
val context = AndroidEnv.context
try {
val file = File(path)
val uri = FileProvider.getUriForFile(context, "${context.packageName}.fileprovider", file)
val intent = Intent(Intent.ACTION_VIEW).apply {
setDataAndType(uri, mime)
addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
}
context.startActivity(intent)
} catch (e: Exception) {
Toast.makeText(context, "No app to open ${mime}", Toast.LENGTH_SHORT).show()
}
}
@@ -2,6 +2,8 @@ package iris.data
import iris.protocol.CommentaryPayload
import iris.protocol.Frame
import iris.protocol.MediaOfferPayload
import iris.protocol.MediaRef
import iris.protocol.MessagePayload
import iris.protocol.MessageStartPayload
import iris.protocol.MessageStopPayload
@@ -9,6 +11,7 @@ import iris.protocol.MessageUpdatePayload
import iris.protocol.ROLE_ASSISTANT
import iris.protocol.ROLE_USER
import iris.protocol.TYPE_COMMENTARY
import iris.protocol.TYPE_MEDIA_OFFER
import iris.protocol.TYPE_MESSAGE
import iris.protocol.TYPE_MESSAGE_START
import iris.protocol.TYPE_MESSAGE_STOP
@@ -51,8 +54,23 @@ data class MessageItem(
val streaming: Boolean = false,
val model: String? = null,
val tokens: Int? = null,
val media: List<MediaItem> = emptyList(),
) : ChatItem
/**
* A media attachment on a message (M4). [localPath] is null until the file
* has been pulled to the local cache (outbound) — inbound attachments the
* app itself uploaded carry no local path (the agent reads the server copy).
*/
data class MediaItem(
val mediaId: String,
val kind: String,
val mime: String,
val size: Long,
val filename: String,
val localPath: String? = null,
)
/** A structured tool-activity card (spinner until [done]). */
data class ToolItem(
override val id: String,
@@ -117,10 +135,12 @@ class ChatStore {
// ── Optimistic send ───────────────────────────────────────────────────
/** Optimistic add: show the user's message immediately (pending) in [lane]. */
fun addPending(text: String, lane: String): String {
fun addPending(text: String, lane: String, media: List<MediaItem> = emptyList()): String {
localSeq++
val id = "local_$localSeq"
updateLane(lane) { it + MessageItem(id = id, role = ROLE_USER, text = text, ts = 0, pending = true) }
updateLane(lane) {
it + MessageItem(id = id, role = ROLE_USER, text = text, ts = 0, pending = true, media = media)
}
return id
}
@@ -138,6 +158,7 @@ class ChatStore {
TYPE_TOOL_PROGRESS -> onToolProgress(lane, frame)
TYPE_TOOL_END -> onToolEnd(lane, frame)
TYPE_COMMENTARY -> onCommentary(lane, frame)
TYPE_MEDIA_OFFER -> onMediaOffer(lane, frame)
else -> Unit
}
}
@@ -149,15 +170,17 @@ class ChatStore {
updateLane(lane) { list ->
val byId = list.indexOfFirst { it.id == p.messageId }
if (byId >= 0) {
val updated = (list[byId] as? MessageItem)?.copy(
val cur = list[byId] as? MessageItem ?: return@updateLane list
val updated = cur.copy(
text = p.text,
reasoning = p.reasoning,
pending = false,
streaming = false,
model = p.model,
tokens = p.tokens,
ts = p.ts ?: 0,
) ?: return@updateLane list
ts = p.ts ?: cur.ts,
media = mergeMedia(cur.media, p.media),
)
list.toMutableList().also { it[byId] = updated }
} else if (p.role == ROLE_USER) {
// Replace the matching optimistic pending bubble (server echo).
@@ -169,23 +192,40 @@ class ChatStore {
it[pendingIdx] = MessageItem(
id = p.messageId, role = p.role, text = p.text,
ts = p.ts ?: 0, reasoning = p.reasoning,
media = mergeMedia((list[pendingIdx] as MessageItem).media, p.media),
)
}
} else {
list + MessageItem(
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
media = p.media.map { it.toMediaItem() },
)
}
} else {
list + MessageItem(
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
media = p.media.map { it.toMediaItem() },
)
}
}
}
/** Merge server media refs into existing items, keeping local paths. */
private fun mergeMedia(existing: List<MediaItem>, incoming: List<MediaRef>): List<MediaItem> {
if (incoming.isEmpty()) return existing
val byId = existing.associateBy { it.mediaId }
return incoming.map { ref ->
byId[ref.mediaId]?.copy(
kind = ref.kind, mime = ref.mime, size = ref.size, filename = ref.filename,
) ?: ref.toMediaItem()
}
}
private fun MediaRef.toMediaItem() =
MediaItem(mediaId = mediaId, kind = kind, mime = mime, size = size, filename = filename)
// ── message.start (open a live streaming bubble) ──────────────────────
private fun onMessageStart(lane: String, frame: Frame) {
@@ -295,6 +335,53 @@ class ChatStore {
}
}
// ── M4: media.offer (agent produced media; pull it) ─────────────────────
/**
* Attach an offered media item to its message (by [MediaOfferPayload.messageId],
* falling back to the lane's last assistant message). The item has no local
* path yet — the controller pulls it and calls [setMediaLocalPath].
*/
private fun onMediaOffer(lane: String, frame: Frame) {
val p = frame.payloadAs<MediaOfferPayload>() ?: return
updateLane(lane) { list ->
// Already attached (offer replayed)? Skip.
if (list.any { it is MessageItem && it.media.any { m -> m.mediaId == p.mediaId } }) return@updateLane list
val item = MediaItem(
mediaId = p.mediaId, kind = p.kind, mime = p.mime,
size = p.size, filename = p.filename,
)
val targetIdx = if (p.messageId != null) {
list.indexOfFirst { it.id == p.messageId }
} else {
list.indexOfLast { it is MessageItem && it.role == ROLE_ASSISTANT }
}
if (targetIdx < 0) return@updateLane list
val cur = list[targetIdx] as? MessageItem ?: return@updateLane list
list.toMutableList().also { it[targetIdx] = cur.copy(media = cur.media + item) }
}
}
/** Record a pulled file's local path for [mediaId] (all lanes). */
fun setMediaLocalPath(mediaId: String, localPath: String) {
val map = _lanes.value.toMutableMap()
var changed = false
for ((lane, list) in map) {
val updated = list.map { item ->
if (item is MessageItem) {
item.copy(media = item.media.map { m ->
if (m.mediaId == mediaId) m.copy(localPath = localPath) else m
})
} else item
}
if (updated != list) {
map[lane] = updated
changed = true
}
}
if (changed) _lanes.value = map
}
fun clear() {
_lanes.value = emptyMap()
}
@@ -0,0 +1,10 @@
package iris.media
/** A readable local file (expect/actual; JVM impl in jvmMain). */
expect class FileSource(path: String) : AutoCloseable {
/** Total size in bytes. */
fun size(): Long
/** Read up to [buf.size] bytes into [buf]; returns bytes read or -1 at EOF. */
fun read(buf: ByteArray): Int
}
@@ -0,0 +1,25 @@
package iris.media
/**
* Local media cache (docs/07 §7.6). Pulled media is written under the
* app-specific cache dir; LRU eviction by total size keeps it bounded
* (default 500 MB).
*/
expect class MediaCache(baseDir: String) {
/** Path for a cached media id (null when not cached). */
fun path(mediaId: String, mime: String): String?
/** Open a writer for [mediaId]; [MediaWriter.path] is the final location. */
fun openWriter(mediaId: String, mime: String): MediaWriter
/** Delete a cached file (e.g. after a failed pull). */
fun remove(mediaId: String, mime: String)
}
/** Streaming writer for a cache entry (write chunks, then close). */
expect interface MediaWriter : AutoCloseable {
/** Final file path (valid after [close]). */
val path: String
fun write(bytes: ByteArray)
}
@@ -0,0 +1,37 @@
package iris.media
import iris.protocol.KIND_AUDIO
import iris.protocol.KIND_DOCUMENT
import iris.protocol.KIND_IMAGE
import iris.protocol.KIND_VIDEO
/** Map a MIME type to a media kind (docs/07 §7.1). */
fun kindFromMime(mime: String): String = when {
mime.startsWith("image/") -> KIND_IMAGE
mime.startsWith("video/") -> KIND_VIDEO
mime.startsWith("audio/") -> KIND_AUDIO
else -> KIND_DOCUMENT
}
/** Best-effort file extension for a MIME type (cache file naming). */
fun extForMime(mime: String): String = when {
mime == "image/jpeg" -> ".jpg"
mime == "image/png" -> ".png"
mime == "image/webp" -> ".webp"
mime == "image/gif" -> ".gif"
mime == "image/heic" -> ".heic"
mime == "image/heif" -> ".heif"
mime == "video/mp4" -> ".mp4"
mime == "video/webm" -> ".webm"
mime == "video/quicktime" -> ".mov"
mime == "audio/mpeg" -> ".mp3"
mime == "audio/mp4" || mime == "audio/x-m4a" -> ".m4a"
mime == "audio/ogg" -> ".ogg"
mime == "audio/wav" -> ".wav"
mime == "audio/flac" -> ".flac"
mime == "audio/aac" -> ".aac"
mime == "application/pdf" -> ".pdf"
mime == "application/zip" -> ".zip"
mime == "text/plain" -> ".txt"
else -> ".bin"
}
@@ -0,0 +1,9 @@
package iris.media
/** Streaming SHA-256 for uploads (hash while chunking, no full-file copy). */
expect class Sha256 constructor() {
fun update(bytes: ByteArray, offset: Int = 0, length: Int = bytes.size - offset)
/** Lowercase hex digest of everything fed so far. */
fun hex(): String
}
@@ -1,21 +1,31 @@
package iris.net
import iris.data.SecureStore
import iris.media.FileSource
import iris.media.Sha256
import iris.protocol.ChannelInfo
import iris.protocol.ErrorPayload
import iris.protocol.Frame
import iris.protocol.HelloAckPayload
import iris.protocol.IrisJson
import iris.protocol.MediaPullEndPayload
import iris.protocol.MediaUploadAckPayload
import iris.protocol.ServerCaps
import iris.protocol.TYPE_ERROR
import iris.protocol.TYPE_HELLO_ACK
import iris.protocol.TYPE_MEDIA_PULL_END
import iris.protocol.TYPE_MEDIA_UPLOAD_ACK
import iris.protocol.TYPE_PONG
import iris.protocol.helloFrame
import iris.protocol.mediaPullFrame
import iris.protocol.mediaUploadEndFrame
import iris.protocol.mediaUploadStartFrame
import iris.protocol.messageSendFrame
import iris.protocol.pingFrame
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
@@ -26,10 +36,15 @@ import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.withTimeoutOrNull
import kotlin.time.TimeMark
import kotlin.time.TimeSource
import okhttp3.OkHttpClient
import okio.ByteString
import okio.ByteString.Companion.toByteString
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
@@ -75,6 +90,23 @@ class GatewayClient(
private var lastLiveness: TimeMark = TimeSource.Monotonic.markNow()
private val pending = mutableMapOf<Int, CompletableDeferred<Frame>>()
// M4: binary frames (media upload chunks / pull stream) have no per-frame
// id, so at most one binary session is active per socket. The gateway
// allows one upload per connection; pull is request/response.
private sealed interface BinarySession {
data class Pulling(
val requestId: Int,
val chunks: Channel<ByteArray>,
val end: CompletableDeferred<Frame>,
) : BinarySession
}
private var binarySession: BinarySession? = null
// M4: only one pull may be in flight at a time (binarySession is a single
// slot). Serialize concurrent offers so their byte streams don't interleave.
private val pullMutex = Mutex()
// ── Lifecycle ─────────────────────────────────────────────────────────
/** Start (or resume) the connect loop. Idempotent. */
@@ -177,11 +209,30 @@ class GatewayClient(
TYPE_PONG -> Unit
else -> {
_events.tryEmit(frame)
frame.id?.let { pending[it]?.complete(frame) }
frame.id?.let { id ->
pending[id]?.complete(frame)
// M4: terminal frame of a pull stream — close
// the chunk channel so the pull loop exits.
if (frame.type == TYPE_MEDIA_PULL_END) {
(binarySession as? BinarySession.Pulling)?.let {
it.chunks.close()
binarySession = null
}
}
}
}
}
}
override fun onMessage(webSocket: WebSocket, bytes: ByteString) {
lastLiveness = TimeSource.Monotonic.markNow()
// M4: binary frames belong to the active pull stream
// (uploads are outbound; stray inbound chunks are dropped).
(binarySession as? BinarySession.Pulling)
?.chunks
?.trySend(bytes.toByteArray())
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
closed.complete(Unit)
}
@@ -215,11 +266,122 @@ class GatewayClient(
// ── Outbound ──────────────────────────────────────────────────────────
/** Send a text message (fire-and-forget; the server echoes it back). */
fun sendMessage(chatId: String, text: String, threadId: String? = null) {
/** Send a text message (fire-and-forget; the server echoes it back).
* M4: [mediaRefs] reference completed uploads (media.upload.ack refs). */
fun sendMessage(
chatId: String,
text: String,
threadId: String? = null,
mediaRefs: List<String> = emptyList(),
) {
val ws = socket ?: return
val id = nextRequestId++
ws.send(messageSendFrame(id, chatId, text, threadId).toWire())
ws.send(messageSendFrame(id, chatId, text, threadId, mediaRefs).toWire())
}
// ── M4: media upload / pull ───────────────────────────────────────────
/**
* Upload a local file as media (docs/07 §7.2): media.upload.start,
* 256 KiB binary chunks, media.upload.end {sha256}. Returns the server's
* media_ref (for message.send media_refs) on success.
*/
suspend fun uploadMedia(
path: String,
mime: String,
kind: String,
filename: String,
mediaRef: String,
): Result<String> {
val ws = socket ?: return Result.failure(IllegalStateException("not connected"))
val source = FileSource(path)
val size = source.size()
if (size <= 0) {
source.close()
return Result.failure(IllegalStateException("empty file"))
}
val id = nextRequestId++
val reply = CompletableDeferred<Frame>()
pending[id] = reply
try {
ws.send(mediaUploadStartFrame(id, mediaRef, kind, mime, filename, size).toWire())
val sha = Sha256()
source.use {
val buf = ByteArray(UPLOAD_CHUNK_BYTES)
while (true) {
val n = it.read(buf)
if (n < 0) break
if (n == 0) continue
sha.update(buf, 0, n)
ws.send(buf.copyOfRange(0, n).toByteString())
}
}
ws.send(mediaUploadEndFrame(id, mediaRef, sha.hex()).toWire())
val frame = withTimeout(UPLOAD_TIMEOUT_MS) { reply.await() }
return when (frame.type) {
TYPE_MEDIA_UPLOAD_ACK -> {
val p = frame.payloadAs<MediaUploadAckPayload>()
if (p != null && p.ok) Result.success(p.mediaRef)
else Result.failure(IllegalStateException("upload rejected by server"))
}
TYPE_ERROR -> {
val e = frame.payloadAs<ErrorPayload>()
Result.failure(IllegalStateException(e?.message ?: "upload failed"))
}
else -> Result.failure(IllegalStateException("unexpected reply ${frame.type}"))
}
} catch (e: Exception) {
return Result.failure(e)
} finally {
pending.remove(id)
}
}
/**
* Pull offered media (docs/07 §7.3): media.pull, then binary frames until
* media.pull.end. Each chunk is handed to [onChunk] (write to cache).
*/
suspend fun pullMedia(mediaId: String, onChunk: suspend (ByteArray) -> Unit): Result<Unit> =
pullMutex.withLock {
val ws = socket ?: return@withLock Result.failure(IllegalStateException("not connected"))
val id = nextRequestId++
val chunks = Channel<ByteArray>(Channel.UNLIMITED)
val end = CompletableDeferred<Frame>()
pending[id] = end
binarySession = BinarySession.Pulling(id, chunks, end)
try {
ws.send(mediaPullFrame(id, mediaId).toWire())
val frame = withTimeout(PULL_TIMEOUT_MS) {
for (chunk in chunks) onChunk(chunk)
end.await()
}
when (frame.type) {
TYPE_MEDIA_PULL_END -> {
val p = frame.payloadAs<MediaPullEndPayload>()
if (p != null && p.ok) Result.success(Unit)
else Result.failure(IllegalStateException("pull failed"))
}
TYPE_ERROR -> {
val e = frame.payloadAs<ErrorPayload>()
Result.failure(IllegalStateException(e?.message ?: "pull failed"))
}
else -> Result.failure(IllegalStateException("unexpected reply ${frame.type}"))
}
} catch (e: Exception) {
Result.failure(e)
} finally {
pending.remove(id)
chunks.cancel()
val s = binarySession
if (s is BinarySession.Pulling && s.requestId == id) binarySession = null
}
}
companion object {
/** One WS binary frame carries at most this many media bytes (docs/07 §7.5). */
const val UPLOAD_CHUNK_BYTES = 256 * 1024
const val UPLOAD_TIMEOUT_MS = 120_000L
const val PULL_TIMEOUT_MS = 300_000L
}
/**
@@ -0,0 +1,34 @@
package iris.platform
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.ImageBitmap
import iris.data.MediaItem
/** A file picked by the user, staged as a local file the app can read. */
data class PickedFile(
val path: String,
val name: String,
val mime: String,
val size: Long,
)
/**
* File picker (SAF on Android, FileChooser on desktop). Invokes [onPicked]
* with the staged file, or null when the user cancels.
*/
@Composable
expect fun MediaFilePicker(onPicked: (PickedFile?) -> Unit)
/** Base directory for the media cache (app-specific cache dir). */
expect fun mediaCacheBaseDir(): String
/** Load an image from a local path for display. Null on failure. */
expect suspend fun loadMediaImage(path: String): ImageBitmap?
/** Inline audio/video player (ExoPlayer on Android; placeholder on desktop). */
@Composable
expect fun MediaPlayerView(media: MediaItem, modifier: Modifier)
/** Open a document with the system handler (best effort). */
expect fun openDocument(path: String, mime: String)
@@ -46,6 +46,14 @@ const val TYPE_TOOL_PROGRESS = "tool.progress"
const val TYPE_TOOL_END = "tool.end"
const val TYPE_COMMENTARY = "commentary"
// M4 — media (upload / offer / pull)
const val TYPE_MEDIA_UPLOAD_START = "media.upload.start"
const val TYPE_MEDIA_UPLOAD_END = "media.upload.end"
const val TYPE_MEDIA_UPLOAD_ACK = "media.upload.ack"
const val TYPE_MEDIA_OFFER = "media.offer"
const val TYPE_MEDIA_PULL = "media.pull"
const val TYPE_MEDIA_PULL_END = "media.pull.end"
// M3 — channels / threads / search / sync
const val TYPE_CHANNEL_CREATE = "channel.create"
const val TYPE_CHANNEL_RENAME = "channel.rename"
@@ -66,6 +74,14 @@ const val ERR_AUTH = "auth"
const val ERR_NOT_FOUND = "not_found"
const val ERR_UNSUPPORTED = "unsupported"
const val ERR_INTERNAL = "internal"
const val ERR_MEDIA_TOO_LARGE = "media_too_large"
// M4 — media kinds (docs/07 §7.1)
const val KIND_IMAGE = "image"
const val KIND_AUDIO = "audio"
const val KIND_VIDEO = "video"
const val KIND_DOCUMENT = "document"
const val KIND_VOICE = "voice"
// ── Roles ───────────────────────────────────────────────────────────────
@@ -147,8 +163,60 @@ data class MessagePayload(
val model: String? = null,
val tokens: Int? = null,
val ts: Long? = null,
val media: List<MediaRef> = emptyList(),
)
// ── M4: media ────────────────────────────────────────────────────────────
/** A media attachment referenced by a message (or offered for pull). */
@Serializable
data class MediaRef(
@SerialName("media_id") val mediaId: String,
val kind: String,
val mime: String,
val size: Long,
val filename: String,
)
@Serializable
data class MediaUploadStartPayload(
@SerialName("media_ref") val mediaRef: String,
val kind: String,
val mime: String,
val filename: String,
val size: Long,
)
@Serializable
data class MediaUploadEndPayload(
@SerialName("media_ref") val mediaRef: String,
@SerialName("sha256") val sha256: String,
)
@Serializable
data class MediaUploadAckPayload(
val ok: Boolean,
@SerialName("media_ref") val mediaRef: String,
)
@Serializable
data class MediaOfferPayload(
@SerialName("media_id") val mediaId: String,
val kind: String,
val mime: String,
val size: Long,
val filename: String,
@SerialName("message_id") val messageId: String? = null,
)
@Serializable
data class MediaPullPayload(
@SerialName("media_id") val mediaId: String,
)
@Serializable
data class MediaPullEndPayload(val ok: Boolean)
// ── M2: streaming frames (server -> app) ────────────────────────────────
@Serializable
@@ -292,7 +360,13 @@ fun helloFrame(token: String, deviceId: String, deviceName: String): Frame =
),
)
fun messageSendFrame(id: Int, chatId: String, text: String, threadId: String? = null): Frame =
fun messageSendFrame(
id: Int,
chatId: String,
text: String,
threadId: String? = null,
mediaRefs: List<String> = emptyList(),
): Frame =
Frame(
id = id,
type = TYPE_MESSAGE_SEND,
@@ -300,7 +374,7 @@ fun messageSendFrame(id: Int, chatId: String, text: String, threadId: String? =
threadId = threadId,
payload = IrisJson.instance.encodeToJsonElement(
MessageSendPayload.serializer(),
MessageSendPayload(text = text),
MessageSendPayload(text = text, mediaRefs = mediaRefs),
),
)
@@ -365,4 +439,43 @@ fun syncFrame(id: Int, cursor: Long): Frame =
SyncPayload.serializer(),
SyncPayload(cursor = cursor),
),
)
// ── M4 frame builders ────────────────────────────────────────────────────
fun mediaUploadStartFrame(
id: Int,
mediaRef: String,
kind: String,
mime: String,
filename: String,
size: Long,
): Frame =
Frame(
id = id,
type = TYPE_MEDIA_UPLOAD_START,
payload = IrisJson.instance.encodeToJsonElement(
MediaUploadStartPayload.serializer(),
MediaUploadStartPayload(mediaRef, kind, mime, filename, size),
),
)
fun mediaUploadEndFrame(id: Int, mediaRef: String, sha256: String): Frame =
Frame(
id = id,
type = TYPE_MEDIA_UPLOAD_END,
payload = IrisJson.instance.encodeToJsonElement(
MediaUploadEndPayload.serializer(),
MediaUploadEndPayload(mediaRef, sha256),
),
)
fun mediaPullFrame(id: Int, mediaId: String): Frame =
Frame(
id = id,
type = TYPE_MEDIA_PULL,
payload = IrisJson.instance.encodeToJsonElement(
MediaPullPayload.serializer(),
MediaPullPayload(mediaId),
),
)
@@ -2,8 +2,14 @@ package iris.state
import iris.data.ChatStore
import iris.data.ChannelStore
import iris.data.MediaItem
import iris.data.SecureStore
import iris.media.MediaCache
import iris.media.kindFromMime
import iris.net.GatewayClient
import iris.platform.PickedFile
import iris.platform.mediaCacheBaseDir
import iris.protocol.MediaOfferPayload
import iris.protocol.SearchHit
import iris.protocol.SearchResultsPayload
import iris.protocol.SyncDonePayload
@@ -12,6 +18,7 @@ import iris.protocol.TYPE_CHANNEL_DELETED
import iris.protocol.TYPE_CHANNEL_LIST
import iris.protocol.TYPE_CHANNEL_RENAMED
import iris.protocol.TYPE_COMMENTARY
import iris.protocol.TYPE_MEDIA_OFFER
import iris.protocol.TYPE_MESSAGE
import iris.protocol.TYPE_MESSAGE_START
import iris.protocol.TYPE_MESSAGE_STOP
@@ -37,6 +44,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlin.random.Random
/**
* App-level controller (M3): owns the GatewayClient + ChatStore + ChannelStore,
@@ -88,6 +96,24 @@ class IrisController(
private val _lastQuery = MutableStateFlow("")
val lastQuery: StateFlow<String> = _lastQuery.asStateFlow()
// ── M4: media ─────────────────────────────────────────────────────────
private val mediaCache = MediaCache(mediaCacheBaseDir())
/** A composer attachment: picked file being uploaded (or uploaded). */
data class PendingAttachment(
val filename: String,
val mime: String,
val size: Long,
val kind: String,
val path: String? = null,
val uploading: Boolean = true,
val mediaRef: String? = null,
val error: String? = null,
)
private val _attachments = MutableStateFlow<List<PendingAttachment>>(emptyList())
val attachments: StateFlow<List<PendingAttachment>> = _attachments.asStateFlow()
init {
scope.launch {
client.events.collect { frame ->
@@ -99,7 +125,14 @@ class IrisController(
TYPE_TOOL_START,
TYPE_TOOL_PROGRESS,
TYPE_TOOL_END,
TYPE_COMMENTARY -> chat.onFrame(frame)
TYPE_COMMENTARY,
TYPE_MEDIA_OFFER -> {
chat.onFrame(frame)
// M4: pull offered media into the local cache.
if (frame.type == TYPE_MEDIA_OFFER) {
frame.payloadAs<MediaOfferPayload>()?.let { pullMedia(it) }
}
}
TYPE_CHANNEL_CREATED,
TYPE_CHANNEL_RENAMED,
TYPE_CHANNEL_DELETED,
@@ -202,14 +235,76 @@ class IrisController(
client.sendFrame(syncFrame(0, cursor))
}
/** Optimistic send: show immediately in the current lane, then hand to the gateway. */
fun send(text: String) {
/** Optimistic send: show immediately in the current lane, then hand to the gateway.
* M4: [attachments] (uploaded) are attached via media_refs. */
fun send(text: String, attachments: List<PendingAttachment> = emptyList()) {
val trimmed = text.trim()
if (trimmed.isEmpty()) return
val refs = attachments.mapNotNull { it.mediaRef }
if (trimmed.isEmpty() && refs.isEmpty()) return
val lane = chat.currentLane.value
val (chatId, threadId) = chat.parseLane(lane)
chat.addPending(trimmed, lane)
client.sendMessage(chatId, trimmed, threadId)
val media = attachments.map {
MediaItem(
mediaId = it.mediaRef ?: it.filename,
kind = it.kind, mime = it.mime, size = it.size, filename = it.filename,
localPath = it.path,
)
}
chat.addPending(trimmed, lane, media)
client.sendMessage(chatId, trimmed, threadId, refs)
_attachments.value = emptyList()
}
// ── M4: attachments (pick -> upload -> send) ──────────────────────────
/** Stage a picked file: upload it, then keep it as a pending attachment. */
fun attachFile(picked: PickedFile) {
val kind = kindFromMime(picked.mime)
val placeholder = PendingAttachment(
filename = picked.name, mime = picked.mime, size = picked.size, kind = kind,
path = picked.path,
)
_attachments.value = _attachments.value + placeholder
scope.launch {
val mediaRef = "up_${Random.nextLong(1_000_000_000L, 9_999_999_999L)}"
val result = client.uploadMedia(
path = picked.path,
mime = picked.mime,
kind = kind,
filename = picked.name,
mediaRef = mediaRef,
)
_attachments.value = _attachments.value.map {
if (it.filename == picked.name && it.uploading) {
result.fold(
{ ref -> it.copy(uploading = false, mediaRef = ref) },
{ e -> it.copy(uploading = false, error = e.message) },
)
} else it
}
}
}
fun removeAttachment(filename: String) {
_attachments.value = _attachments.value.filterNot { it.filename == filename }
}
/** Pull offered media into the local cache and record the path. */
private fun pullMedia(offer: MediaOfferPayload) {
scope.launch {
// Already cached? Skip the pull.
mediaCache.path(offer.mediaId, offer.mime)?.let {
chat.setMediaLocalPath(offer.mediaId, it)
return@launch
}
val writer = mediaCache.openWriter(offer.mediaId, offer.mime)
val result = client.pullMedia(offer.mediaId) { chunk -> writer.write(chunk) }
writer.close()
result.fold(
{ chat.setMediaLocalPath(offer.mediaId, writer.path) },
{ mediaCache.remove(offer.mediaId, offer.mime) },
)
}
}
/** Connect-screen action: real hello test, then save + (re)connect. */
@@ -16,6 +16,8 @@ import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.Image
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
@@ -47,7 +49,10 @@ import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalClipboardManager
import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
@@ -56,10 +61,17 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import iris.data.ChannelStore
import iris.data.ChatItem
import iris.data.MediaItem
import iris.data.MessageItem
import iris.data.ToolItem
import iris.net.GatewayClient
import iris.platform.MediaFilePicker
import iris.platform.MediaPlayerView
import iris.platform.loadMediaImage
import iris.platform.openDocument
import iris.protocol.ChannelInfo
import iris.protocol.KIND_DOCUMENT
import iris.protocol.KIND_IMAGE
import iris.protocol.ROLE_USER
import iris.protocol.SearchHit
import iris.state.IrisController
@@ -91,14 +103,20 @@ fun ChatScreen(controller: IrisController) {
var showSearch by remember { mutableStateOf(false) }
var showNewChannel by remember { mutableStateOf(false) }
var showNewThread by remember { mutableStateOf(false) }
var showPicker by remember { mutableStateOf(false) }
val attachments by controller.attachments.collectAsState()
val drawerState = rememberDrawerState(DrawerValue.Closed)
val drawerScope = rememberCoroutineScope()
val focusManager = LocalFocusManager.current
fun doSend() {
if (input.isBlank()) return
val ready = attachments.filter { it.mediaRef != null && it.error == null }
if (input.isBlank() && ready.isEmpty()) return
val text = input
input = ""
controller.send(text)
controller.send(text, ready)
// Best practice: dismiss the keyboard after sending.
focusManager.clearFocus(force = true)
}
LaunchedEffect(items.size, typing) {
@@ -206,6 +224,21 @@ fun ChatScreen(controller: IrisController) {
}
}
// Pending attachments (M4)
if (attachments.isNotEmpty()) {
Row(
modifier = Modifier
.fillMaxWidth()
.horizontalScroll(rememberScrollState())
.padding(horizontal = 12.dp, vertical = 4.dp),
horizontalArrangement = Arrangement.spacedBy(6.dp),
) {
attachments.forEach { att ->
AttachmentChip(att, onRemove = { controller.removeAttachment(att.filename) })
}
}
}
// Composer
Row(
modifier = Modifier
@@ -213,6 +246,9 @@ fun ChatScreen(controller: IrisController) {
.padding(12.dp),
verticalAlignment = Alignment.Bottom,
) {
IconButton(onClick = { showPicker = true }) {
Text("📎", fontSize = 18.sp)
}
OutlinedTextField(
value = input,
onValueChange = { input = it },
@@ -226,7 +262,7 @@ fun ChatScreen(controller: IrisController) {
Spacer(modifier = Modifier.width(8.dp))
Button(
onClick = { doSend() },
enabled = input.isNotBlank(),
enabled = input.isNotBlank() || attachments.any { it.mediaRef != null && it.error == null },
) {
Text("Send")
}
@@ -234,6 +270,16 @@ fun ChatScreen(controller: IrisController) {
}
}
// File picker overlay (M4)
if (showPicker) {
Box(modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.5f))) {
MediaFilePicker(onPicked = { picked ->
showPicker = false
if (picked != null) controller.attachFile(picked)
})
}
}
// Search overlay
if (showSearch) {
SearchOverlay(
@@ -520,6 +566,13 @@ private fun MessageBubble(msg: MessageItem) {
fontSize = if (isCommentary) 13.sp else 15.sp,
)
}
// M4: media attachments (image / player / document chip).
if (msg.media.isNotEmpty()) {
Spacer(modifier = Modifier.height(6.dp))
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
msg.media.forEach { m -> MediaAttachment(m) }
}
}
if (msg.pending) {
Text("sending…", color = textColor.copy(alpha = 0.6f), fontSize = 10.sp)
}
@@ -542,6 +595,135 @@ private fun MessageBubble(msg: MessageItem) {
}
}
/**
* M4: render one media attachment in a bubble.
* - image: tappable thumbnail (full-screen viewer)
* - audio/video: inline player (ExoPlayer on Android)
* - document: filename chip (tap to open with system handler)
*/
@Composable
private fun MediaAttachment(media: MediaItem) {
when (media.kind) {
KIND_IMAGE -> MediaImage(media)
KIND_DOCUMENT -> MediaDocChip(media)
else -> MediaPlayerView(media, Modifier.fillMaxWidth())
}
}
@Composable
private fun MediaImage(media: MediaItem) {
var bitmap by remember(media.localPath) { mutableStateOf<ImageBitmap?>(null) }
var loading by remember(media.localPath) { mutableStateOf(media.localPath != null) }
var showFull by remember { mutableStateOf(false) }
LaunchedEffect(media.localPath) {
val path = media.localPath
if (path != null) {
bitmap = loadMediaImage(path)
loading = false
}
}
if (loading || bitmap == null) {
Box(
modifier = Modifier
.size(120.dp)
.clip(RoundedCornerShape(10.dp))
.background(Color(0xFF20242E)),
contentAlignment = Alignment.Center,
) {
if (media.localPath == null) Text("downloading…", fontSize = 11.sp)
else CircularProgressIndicator(modifier = Modifier.size(20.dp), strokeWidth = 2.dp)
}
return
}
val bmp = bitmap!!
val aspect = bmp.width.toFloat() / bmp.height.toFloat()
// Preserve the image's aspect ratio, capping the dominant dimension at
// 260.dp so a chat image never dominates the screen (the other dimension
// follows the ratio).
val (imgW, imgH) = if (aspect >= 1f) 260.dp to (260 / aspect).dp
else (260 * aspect).dp to 260.dp
Image(
bitmap = bmp,
contentDescription = media.filename,
contentScale = ContentScale.FillBounds,
modifier = Modifier
.width(imgW)
.height(imgH)
.clip(RoundedCornerShape(10.dp))
.clickable { showFull = true },
)
if (showFull) {
Box(
modifier = Modifier
.fillMaxSize()
.background(Color.Black)
.clickable { showFull = false },
contentAlignment = Alignment.Center,
) {
Image(
bitmap = bitmap!!,
contentDescription = media.filename,
contentScale = ContentScale.Fit,
modifier = Modifier
.fillMaxWidth()
.fillMaxHeight(0.9f),
)
}
}
}
@Composable
private fun MediaDocChip(media: MediaItem) {
Row(
modifier = Modifier
.clip(RoundedCornerShape(10.dp))
.background(Color(0xFF20242E))
.clickable {
media.localPath?.let { openDocument(it, media.mime) }
}
.padding(horizontal = 10.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Text("📄", fontSize = 16.sp)
Spacer(modifier = Modifier.width(8.dp))
Column {
Text(media.filename, fontSize = 13.sp, maxLines = 1)
Text(fmtSize(media.size), fontSize = 10.sp, color = Color(0xFF8A93A6))
}
}
}
private fun fmtSize(bytes: Long): String = when {
bytes >= 1_048_576 -> "%.1f MB".format(bytes / 1_048_576.0)
bytes >= 1024 -> "%.0f KB".format(bytes / 1024.0)
else -> "$bytes B"
}
/** M4: composer attachment chip (uploading / ready / error). */
@Composable
private fun AttachmentChip(att: IrisController.PendingAttachment, onRemove: () -> Unit) {
val (icon, label) = when {
att.error != null -> "⚠️" to "${att.filename} — ${att.error}"
att.uploading -> "⏳" to "${att.filename} (${fmtSize(att.size)})"
else -> "📎" to "${att.filename} (${fmtSize(att.size)})"
}
Row(
modifier = Modifier
.clip(RoundedCornerShape(12.dp))
.background(Color(0xFF2A2E3B))
.padding(horizontal = 8.dp, vertical = 5.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Text(icon, fontSize = 12.sp)
Spacer(modifier = Modifier.width(4.dp))
Text(label, fontSize = 12.sp, color = Color(0xFFC7CCD8), maxLines = 1)
Spacer(modifier = Modifier.width(4.dp))
TextButton(onClick = onRemove) { Text("✕", fontSize = 11.sp) }
}
}
/**
* Collapsible reasoning panel (M2): header "💭 Reasoning", monospace body,
* copy button. Collapsed by default when long; tap to toggle.
@@ -0,0 +1,115 @@
package iris.platform
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.material3.Button
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import iris.data.MediaItem
import java.io.File
import javax.imageio.ImageIO
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@Composable
actual fun MediaFilePicker(onPicked: (PickedFile?) -> Unit) {
var path by remember { mutableStateOf("") }
Column(
modifier = Modifier
.fillMaxWidth()
.padding(16.dp),
) {
Text("File path (desktop):", fontSize = 13.sp)
Spacer(modifier = Modifier.width(0.dp))
OutlinedTextField(
value = path,
onValueChange = { path = it },
modifier = Modifier.fillMaxWidth(),
placeholder = { Text("/path/to/file") },
)
Spacer(modifier = Modifier.width(0.dp))
Row {
Button(onClick = {
val f = File(path.trim())
if (f.exists() && f.isFile) {
onPicked(
PickedFile(
path = f.absolutePath,
name = f.name,
mime = guessMime(f.name),
size = f.length(),
),
)
} else {
onPicked(null)
}
}, enabled = path.isNotBlank()) {
Text("Attach")
}
}
}
}
private fun guessMime(name: String): String {
val ext = name.substringAfterLast('.', "").lowercase()
return when (ext) {
"jpg", "jpeg" -> "image/jpeg"
"png" -> "image/png"
"webp" -> "image/webp"
"gif" -> "image/gif"
"mp4" -> "video/mp4"
"webm" -> "video/webm"
"mov" -> "video/quicktime"
"mp3" -> "audio/mpeg"
"m4a", "aac" -> "audio/mp4"
"ogg", "opus" -> "audio/ogg"
"wav" -> "audio/wav"
"pdf" -> "application/pdf"
"txt" -> "text/plain"
else -> "application/octet-stream"
}
}
actual fun mediaCacheBaseDir(): String {
val dir = File(System.getProperty("user.home"), ".iris/cache")
dir.mkdirs()
return dir.absolutePath
}
actual suspend fun loadMediaImage(path: String): ImageBitmap? {
// Desktop image decoding via BufferedImage->ImageBitmap is a later
// milestone; return null so bubbles show the "loading" placeholder.
return null
}
@Composable
actual fun MediaPlayerView(media: MediaItem, modifier: Modifier) {
// Desktop playback backend (libmpv/WebView) is a later milestone;
// show a chip with the file info for now.
Box(modifier = modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
Text(
"${media.kind}: ${media.filename} (${media.size} B) — playback not available on desktop yet",
fontSize = 12.sp,
)
}
}
actual fun openDocument(path: String, mime: String) {
// Desktop open-with is a later milestone; no-op for now.
}
@@ -0,0 +1,16 @@
package iris.media
import java.io.File
actual class FileSource actual constructor(path: String) : AutoCloseable {
private val file = File(path)
private val input = file.inputStream()
actual fun size(): Long = file.length()
actual fun read(buf: ByteArray): Int = input.read(buf)
override fun close() {
input.close()
}
}
@@ -0,0 +1,59 @@
package iris.media
import java.io.File
import java.io.FileOutputStream
actual interface MediaWriter : AutoCloseable {
actual val path: String
actual fun write(bytes: ByteArray)
}
actual class MediaCache actual constructor(baseDir: String) {
private val maxBytes: Long = 500L * 1024 * 1024
private val mediaDir: File = File(baseDir, "media").apply { mkdirs() }
actual fun path(mediaId: String, mime: String): String? {
val f = File(mediaDir, "$mediaId${extForMime(mime)}")
return if (f.exists()) f.absolutePath else null
}
actual fun openWriter(mediaId: String, mime: String): MediaWriter =
JvmMediaWriter(File(mediaDir, "$mediaId${extForMime(mime)}"), this)
actual fun remove(mediaId: String, mime: String) {
path(mediaId, mime)?.let { File(it).delete() }
}
internal fun evict() {
val files = mediaDir.listFiles()?.filter { !it.name.endsWith(".part") } ?: return
var total = files.sumOf { it.length() }
if (total <= maxBytes) return
files.sortedBy { it.lastModified() }.forEach { f ->
if (total <= maxBytes) return
total -= f.length()
f.delete()
}
}
}
private class JvmMediaWriter(private val target: File, private val cache: MediaCache) : MediaWriter {
private val out: FileOutputStream = FileOutputStream(File(target.parentFile, "${target.name}.part"))
override val path: String get() = target.absolutePath
override fun write(bytes: ByteArray) {
out.write(bytes)
}
override fun close() {
out.close()
val part = File(target.parentFile, "${target.name}.part")
if (target.exists()) target.delete()
if (!part.renameTo(target)) {
// Rare cross-device rename failure: fall back to copy.
part.copyTo(target, overwrite = true)
part.delete()
}
cache.evict()
}
}
@@ -0,0 +1,18 @@
package iris.media
import java.security.MessageDigest
actual class Sha256 actual constructor() {
private val digest = MessageDigest.getInstance("SHA-256")
actual fun update(bytes: ByteArray, offset: Int, length: Int) {
digest.update(bytes, offset, length)
}
actual fun hex(): String {
val bytes = digest.digest()
return buildString(bytes.size * 2) {
for (b in bytes) append("%02x".format(b))
}
}
}