M4: media upload/download/playback (both directions)
Gateway plugin: - media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff -> cache_*_from_bytes -> media.upload.ack - media.offer / media.pull (chunked) for agent-sent media, delivery-path security re-checked at pull time - send_* overrides mint media_id and emit media.offer - message.send media_refs resolve to cached inbound media - per-send + per-chunk timeouts so a stalled peer can't starve the rest App (Kotlin CMP): - Protocol: media frame types/payloads/builders - GatewayClient: binary session, uploadMedia (chunked + streaming sha256), pullMedia serialized via Mutex so concurrent offers don't interleave - ChatStore/IrisController: MediaItem, attachments, auto-pull on offer - Platform media: SAF picker, ExoPlayer (audio mini-player + video), image loader, FileProvider document open (Android); AWT-free desktop actuals - ChatScreen: attach button + chips, media rendering, keyboard dismiss on send UI polish: - preserve image aspect ratio (no stretching), cap dominant dimension - adjustResize so only chat content squeezes for the keyboard - clear focus (hide keyboard) on send Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json + 07-media.md; M4 marked complete in 14-milestones.md. Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
1 parent
60296b33fe
commit
913ee91024
28 files changed
+2234
-53
No files matched your search
@@ -12,12 +12,24 @@
|
|||||||
<activity
|
<activity
|
||||||
android:name=".MainActivity"
|
android:name=".MainActivity"
|
||||||
android:exported="true"
|
android:exported="true"
|
||||||
|
android:windowSoftInputMode="adjustResize"
|
||||||
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden">
|
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden">
|
||||||
<intent-filter>
|
<intent-filter>
|
||||||
<action android:name="android.intent.action.MAIN" />
|
<action android:name="android.intent.action.MAIN" />
|
||||||
<category android:name="android.intent.category.LAUNCHER" />
|
<category android:name="android.intent.category.LAUNCHER" />
|
||||||
</intent-filter>
|
</intent-filter>
|
||||||
</activity>
|
</activity>
|
||||||
|
|
||||||
|
<!-- M4: serve cached media/documents to other apps (ACTION_VIEW). -->
|
||||||
|
<provider
|
||||||
|
android:name="androidx.core.content.FileProvider"
|
||||||
|
android:authorities="${applicationId}.fileprovider"
|
||||||
|
android:exported="false"
|
||||||
|
android:grantUriPermissions="true">
|
||||||
|
<meta-data
|
||||||
|
android:name="android.support.FILE_PROVIDER_PATHS"
|
||||||
|
android:resource="@xml/iris_file_paths" />
|
||||||
|
</provider>
|
||||||
</application>
|
</application>
|
||||||
|
|
||||||
</manifest>
|
</manifest>
|
||||||
@@ -4,11 +4,13 @@ import android.os.Bundle
|
|||||||
import androidx.activity.ComponentActivity
|
import androidx.activity.ComponentActivity
|
||||||
import androidx.activity.compose.setContent
|
import androidx.activity.compose.setContent
|
||||||
import iris.IrisApp
|
import iris.IrisApp
|
||||||
|
import iris.platform.AndroidEnv
|
||||||
import iris.platform.AndroidSecureStore
|
import iris.platform.AndroidSecureStore
|
||||||
|
|
||||||
class MainActivity : ComponentActivity() {
|
class MainActivity : ComponentActivity() {
|
||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
super.onCreate(savedInstanceState)
|
super.onCreate(savedInstanceState)
|
||||||
|
AndroidEnv.context = applicationContext
|
||||||
val store = AndroidSecureStore(applicationContext)
|
val store = AndroidSecureStore(applicationContext)
|
||||||
setContent {
|
setContent {
|
||||||
IrisApp(store)
|
IrisApp(store)
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<paths>
|
||||||
|
<!-- M4: pulled media lives in externalCacheDir/media -->
|
||||||
|
<external-cache-path name="media" path="media/" />
|
||||||
|
<!-- M4: picked-file staging lives in cacheDir/uploads -->
|
||||||
|
<cache-path name="uploads" path="uploads/" />
|
||||||
|
</paths>
|
||||||
@@ -16,6 +16,12 @@ kotlin {
|
|||||||
jvm("desktop")
|
jvm("desktop")
|
||||||
|
|
||||||
sourceSets {
|
sourceSets {
|
||||||
|
// Both targets are JVM-based (androidTarget + jvm("desktop")), so
|
||||||
|
// shared JVM code (File I/O, SHA-256, media cache) lives in jvmMain.
|
||||||
|
val jvmMain by creating { dependsOn(commonMain.get()) }
|
||||||
|
val androidMain by getting { dependsOn(jvmMain) }
|
||||||
|
val desktopMain by getting { dependsOn(jvmMain) }
|
||||||
|
|
||||||
commonMain.dependencies {
|
commonMain.dependencies {
|
||||||
implementation("org.jetbrains.compose.runtime:runtime:$composeVersion")
|
implementation("org.jetbrains.compose.runtime:runtime:$composeVersion")
|
||||||
implementation("org.jetbrains.compose.foundation:foundation:$composeVersion")
|
implementation("org.jetbrains.compose.foundation:foundation:$composeVersion")
|
||||||
@@ -27,6 +33,13 @@ kotlin {
|
|||||||
// (androidTarget + jvm("desktop")), so it resolves on both.
|
// (androidTarget + jvm("desktop")), so it resolves on both.
|
||||||
implementation("com.squareup.okhttp3:okhttp:$okhttpVersion")
|
implementation("com.squareup.okhttp3:okhttp:$okhttpVersion")
|
||||||
}
|
}
|
||||||
|
// M4: ExoPlayer (Media3) for inline audio/video playback (Android only).
|
||||||
|
androidMain.dependencies {
|
||||||
|
implementation("androidx.media3:media3-exoplayer:1.3.1")
|
||||||
|
implementation("androidx.media3:media3-ui:1.3.1")
|
||||||
|
// SAF picker (rememberLauncherForActivityResult).
|
||||||
|
implementation("androidx.activity:activity-compose:1.9.2")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
package iris.platform
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.content.Intent
|
||||||
|
import android.graphics.BitmapFactory
|
||||||
|
import android.net.Uri
|
||||||
|
import android.provider.OpenableColumns
|
||||||
|
import android.widget.Toast
|
||||||
|
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||||
|
import androidx.activity.result.contract.ActivityResultContracts
|
||||||
|
import androidx.compose.foundation.layout.Box
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.Spacer
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.height
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.foundation.layout.size
|
||||||
|
import androidx.compose.material3.Button
|
||||||
|
import androidx.compose.material3.CircularProgressIndicator
|
||||||
|
import androidx.compose.material3.IconButton
|
||||||
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.Slider
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.DisposableEffect
|
||||||
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.mutableFloatStateOf
|
||||||
|
import androidx.compose.runtime.mutableStateOf
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.runtime.setValue
|
||||||
|
import androidx.compose.ui.Alignment
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.graphics.ImageBitmap
|
||||||
|
import androidx.compose.ui.graphics.asImageBitmap
|
||||||
|
import androidx.compose.ui.platform.LocalContext
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
import androidx.compose.ui.unit.sp
|
||||||
|
import androidx.compose.ui.viewinterop.AndroidView
|
||||||
|
import androidx.core.content.FileProvider
|
||||||
|
import androidx.core.net.toUri
|
||||||
|
import androidx.media3.common.MediaItem as M3MediaItem
|
||||||
|
import androidx.media3.common.Player
|
||||||
|
import androidx.media3.exoplayer.ExoPlayer
|
||||||
|
import androidx.media3.ui.PlayerView
|
||||||
|
import iris.data.MediaItem
|
||||||
|
import iris.media.extForMime
|
||||||
|
import iris.protocol.KIND_VIDEO
|
||||||
|
import java.io.File
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
|
||||||
|
/** App context holder (set in MainActivity.onCreate). */
|
||||||
|
object AndroidEnv {
|
||||||
|
lateinit var context: Context
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
actual fun MediaFilePicker(onPicked: (PickedFile?) -> Unit) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
val launcher = rememberLauncherForActivityResult(ActivityResultContracts.GetContent()) { uri ->
|
||||||
|
if (uri == null) {
|
||||||
|
onPicked(null)
|
||||||
|
return@rememberLauncherForActivityResult
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
val mime = context.contentResolver.getType(uri) ?: "application/octet-stream"
|
||||||
|
val name = queryDisplayName(context, uri) ?: "upload"
|
||||||
|
val dir = File(context.cacheDir, "uploads").apply { mkdirs() }
|
||||||
|
val target = File(dir, "${System.currentTimeMillis()}${extForMime(mime)}")
|
||||||
|
context.contentResolver.openInputStream(uri)!!.use { input ->
|
||||||
|
target.outputStream().use { output -> input.copyTo(output) }
|
||||||
|
}
|
||||||
|
onPicked(PickedFile(target.absolutePath, name, mime, target.length()))
|
||||||
|
} catch (e: Exception) {
|
||||||
|
onPicked(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Box(modifier = Modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
|
||||||
|
Button(onClick = { launcher.launch("*/*") }) { Text("Pick a file") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun queryDisplayName(context: Context, uri: Uri): String? {
|
||||||
|
context.contentResolver.query(uri, null, null, null, null)?.use { c ->
|
||||||
|
val idx = c.getColumnIndex(OpenableColumns.DISPLAY_NAME)
|
||||||
|
if (idx >= 0 && c.moveToFirst()) return c.getString(idx)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun mediaCacheBaseDir(): String =
|
||||||
|
AndroidEnv.context.externalCacheDir?.absolutePath
|
||||||
|
?: AndroidEnv.context.cacheDir.absolutePath
|
||||||
|
|
||||||
|
actual suspend fun loadMediaImage(path: String): ImageBitmap? = withContext(Dispatchers.IO) {
|
||||||
|
try {
|
||||||
|
BitmapFactory.decodeFile(path)?.asImageBitmap()
|
||||||
|
} catch (e: Exception) {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
actual fun MediaPlayerView(media: MediaItem, modifier: Modifier) {
|
||||||
|
val path = media.localPath
|
||||||
|
if (path == null) {
|
||||||
|
Box(modifier = modifier, contentAlignment = Alignment.Center) {
|
||||||
|
Row {
|
||||||
|
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
|
||||||
|
Spacer(modifier = Modifier.size(8.dp))
|
||||||
|
Text("downloading…", fontSize = 12.sp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (media.kind == KIND_VIDEO) {
|
||||||
|
VideoPlayerView(path, modifier)
|
||||||
|
} else {
|
||||||
|
AudioPlayerView(path, media.filename, modifier)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun VideoPlayerView(path: String, modifier: Modifier) {
|
||||||
|
AndroidView(
|
||||||
|
factory = { ctx ->
|
||||||
|
PlayerView(ctx).apply {
|
||||||
|
useController = true
|
||||||
|
val player = ExoPlayer.Builder(ctx).build()
|
||||||
|
player.setMediaItem(M3MediaItem.fromUri(path.toUri()))
|
||||||
|
player.prepare()
|
||||||
|
player.playWhenReady = true
|
||||||
|
this.player = player
|
||||||
|
}
|
||||||
|
},
|
||||||
|
update = {},
|
||||||
|
onRelease = { view -> view.player?.release() },
|
||||||
|
modifier = modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.height(220.dp),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun AudioPlayerView(path: String, filename: String, modifier: Modifier) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
val player = remember {
|
||||||
|
ExoPlayer.Builder(context).build().apply {
|
||||||
|
setMediaItem(M3MediaItem.fromUri(path.toUri()))
|
||||||
|
prepare()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DisposableEffect(player) { onDispose { player.release() } }
|
||||||
|
|
||||||
|
var playing by remember { mutableStateOf(false) }
|
||||||
|
var position by remember { mutableFloatStateOf(0f) }
|
||||||
|
var duration by remember { mutableFloatStateOf(0f) }
|
||||||
|
|
||||||
|
LaunchedEffect(player) {
|
||||||
|
while (true) {
|
||||||
|
val d = player.duration
|
||||||
|
duration = if (d > 0) d / 1000f else 0f
|
||||||
|
position = player.currentPosition / 1000f
|
||||||
|
playing = player.isPlaying
|
||||||
|
kotlinx.coroutines.delay(500)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Row(
|
||||||
|
modifier = modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.padding(8.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
) {
|
||||||
|
IconButton(onClick = {
|
||||||
|
if (playing) player.pause() else player.play()
|
||||||
|
}) {
|
||||||
|
Text(if (playing) "⏸" else "▶️", fontSize = 18.sp)
|
||||||
|
}
|
||||||
|
Column(modifier = Modifier.weight(1f).padding(horizontal = 8.dp)) {
|
||||||
|
Text(filename, fontSize = 12.sp, maxLines = 1)
|
||||||
|
Slider(
|
||||||
|
value = if (duration > 0f) (position / duration).coerceIn(0f, 1f) else 0f,
|
||||||
|
onValueChange = { frac ->
|
||||||
|
player.seekTo((frac * duration * 1000).toLong())
|
||||||
|
},
|
||||||
|
modifier = Modifier.fillMaxWidth(),
|
||||||
|
)
|
||||||
|
Text(
|
||||||
|
"${fmt(position)} / ${fmt(duration)}",
|
||||||
|
fontSize = 10.sp,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun fmt(seconds: Float): String {
|
||||||
|
val s = seconds.toLong()
|
||||||
|
return "%d:%02d".format(s / 60, s % 60)
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun openDocument(path: String, mime: String) {
|
||||||
|
val context = AndroidEnv.context
|
||||||
|
try {
|
||||||
|
val file = File(path)
|
||||||
|
val uri = FileProvider.getUriForFile(context, "${context.packageName}.fileprovider", file)
|
||||||
|
val intent = Intent(Intent.ACTION_VIEW).apply {
|
||||||
|
setDataAndType(uri, mime)
|
||||||
|
addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
|
||||||
|
}
|
||||||
|
context.startActivity(intent)
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Toast.makeText(context, "No app to open ${mime}", Toast.LENGTH_SHORT).show()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,8 @@ package iris.data
|
|||||||
|
|
||||||
import iris.protocol.CommentaryPayload
|
import iris.protocol.CommentaryPayload
|
||||||
import iris.protocol.Frame
|
import iris.protocol.Frame
|
||||||
|
import iris.protocol.MediaOfferPayload
|
||||||
|
import iris.protocol.MediaRef
|
||||||
import iris.protocol.MessagePayload
|
import iris.protocol.MessagePayload
|
||||||
import iris.protocol.MessageStartPayload
|
import iris.protocol.MessageStartPayload
|
||||||
import iris.protocol.MessageStopPayload
|
import iris.protocol.MessageStopPayload
|
||||||
@@ -9,6 +11,7 @@ import iris.protocol.MessageUpdatePayload
|
|||||||
import iris.protocol.ROLE_ASSISTANT
|
import iris.protocol.ROLE_ASSISTANT
|
||||||
import iris.protocol.ROLE_USER
|
import iris.protocol.ROLE_USER
|
||||||
import iris.protocol.TYPE_COMMENTARY
|
import iris.protocol.TYPE_COMMENTARY
|
||||||
|
import iris.protocol.TYPE_MEDIA_OFFER
|
||||||
import iris.protocol.TYPE_MESSAGE
|
import iris.protocol.TYPE_MESSAGE
|
||||||
import iris.protocol.TYPE_MESSAGE_START
|
import iris.protocol.TYPE_MESSAGE_START
|
||||||
import iris.protocol.TYPE_MESSAGE_STOP
|
import iris.protocol.TYPE_MESSAGE_STOP
|
||||||
@@ -51,8 +54,23 @@ data class MessageItem(
|
|||||||
val streaming: Boolean = false,
|
val streaming: Boolean = false,
|
||||||
val model: String? = null,
|
val model: String? = null,
|
||||||
val tokens: Int? = null,
|
val tokens: Int? = null,
|
||||||
|
val media: List<MediaItem> = emptyList(),
|
||||||
) : ChatItem
|
) : ChatItem
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A media attachment on a message (M4). [localPath] is null until the file
|
||||||
|
* has been pulled to the local cache (outbound) — inbound attachments the
|
||||||
|
* app itself uploaded carry no local path (the agent reads the server copy).
|
||||||
|
*/
|
||||||
|
data class MediaItem(
|
||||||
|
val mediaId: String,
|
||||||
|
val kind: String,
|
||||||
|
val mime: String,
|
||||||
|
val size: Long,
|
||||||
|
val filename: String,
|
||||||
|
val localPath: String? = null,
|
||||||
|
)
|
||||||
|
|
||||||
/** A structured tool-activity card (spinner until [done]). */
|
/** A structured tool-activity card (spinner until [done]). */
|
||||||
data class ToolItem(
|
data class ToolItem(
|
||||||
override val id: String,
|
override val id: String,
|
||||||
@@ -117,10 +135,12 @@ class ChatStore {
|
|||||||
// ── Optimistic send ───────────────────────────────────────────────────
|
// ── Optimistic send ───────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Optimistic add: show the user's message immediately (pending) in [lane]. */
|
/** Optimistic add: show the user's message immediately (pending) in [lane]. */
|
||||||
fun addPending(text: String, lane: String): String {
|
fun addPending(text: String, lane: String, media: List<MediaItem> = emptyList()): String {
|
||||||
localSeq++
|
localSeq++
|
||||||
val id = "local_$localSeq"
|
val id = "local_$localSeq"
|
||||||
updateLane(lane) { it + MessageItem(id = id, role = ROLE_USER, text = text, ts = 0, pending = true) }
|
updateLane(lane) {
|
||||||
|
it + MessageItem(id = id, role = ROLE_USER, text = text, ts = 0, pending = true, media = media)
|
||||||
|
}
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,6 +158,7 @@ class ChatStore {
|
|||||||
TYPE_TOOL_PROGRESS -> onToolProgress(lane, frame)
|
TYPE_TOOL_PROGRESS -> onToolProgress(lane, frame)
|
||||||
TYPE_TOOL_END -> onToolEnd(lane, frame)
|
TYPE_TOOL_END -> onToolEnd(lane, frame)
|
||||||
TYPE_COMMENTARY -> onCommentary(lane, frame)
|
TYPE_COMMENTARY -> onCommentary(lane, frame)
|
||||||
|
TYPE_MEDIA_OFFER -> onMediaOffer(lane, frame)
|
||||||
else -> Unit
|
else -> Unit
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -149,15 +170,17 @@ class ChatStore {
|
|||||||
updateLane(lane) { list ->
|
updateLane(lane) { list ->
|
||||||
val byId = list.indexOfFirst { it.id == p.messageId }
|
val byId = list.indexOfFirst { it.id == p.messageId }
|
||||||
if (byId >= 0) {
|
if (byId >= 0) {
|
||||||
val updated = (list[byId] as? MessageItem)?.copy(
|
val cur = list[byId] as? MessageItem ?: return@updateLane list
|
||||||
|
val updated = cur.copy(
|
||||||
text = p.text,
|
text = p.text,
|
||||||
reasoning = p.reasoning,
|
reasoning = p.reasoning,
|
||||||
pending = false,
|
pending = false,
|
||||||
streaming = false,
|
streaming = false,
|
||||||
model = p.model,
|
model = p.model,
|
||||||
tokens = p.tokens,
|
tokens = p.tokens,
|
||||||
ts = p.ts ?: 0,
|
ts = p.ts ?: cur.ts,
|
||||||
) ?: return@updateLane list
|
media = mergeMedia(cur.media, p.media),
|
||||||
|
)
|
||||||
list.toMutableList().also { it[byId] = updated }
|
list.toMutableList().also { it[byId] = updated }
|
||||||
} else if (p.role == ROLE_USER) {
|
} else if (p.role == ROLE_USER) {
|
||||||
// Replace the matching optimistic pending bubble (server echo).
|
// Replace the matching optimistic pending bubble (server echo).
|
||||||
@@ -169,23 +192,40 @@ class ChatStore {
|
|||||||
it[pendingIdx] = MessageItem(
|
it[pendingIdx] = MessageItem(
|
||||||
id = p.messageId, role = p.role, text = p.text,
|
id = p.messageId, role = p.role, text = p.text,
|
||||||
ts = p.ts ?: 0, reasoning = p.reasoning,
|
ts = p.ts ?: 0, reasoning = p.reasoning,
|
||||||
|
media = mergeMedia((list[pendingIdx] as MessageItem).media, p.media),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
list + MessageItem(
|
list + MessageItem(
|
||||||
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
|
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
|
||||||
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
|
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
|
||||||
|
media = p.media.map { it.toMediaItem() },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
list + MessageItem(
|
list + MessageItem(
|
||||||
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
|
id = p.messageId, role = p.role, text = p.text, ts = p.ts ?: 0,
|
||||||
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
|
reasoning = p.reasoning, model = p.model, tokens = p.tokens,
|
||||||
|
media = p.media.map { it.toMediaItem() },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Merge server media refs into existing items, keeping local paths. */
|
||||||
|
private fun mergeMedia(existing: List<MediaItem>, incoming: List<MediaRef>): List<MediaItem> {
|
||||||
|
if (incoming.isEmpty()) return existing
|
||||||
|
val byId = existing.associateBy { it.mediaId }
|
||||||
|
return incoming.map { ref ->
|
||||||
|
byId[ref.mediaId]?.copy(
|
||||||
|
kind = ref.kind, mime = ref.mime, size = ref.size, filename = ref.filename,
|
||||||
|
) ?: ref.toMediaItem()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun MediaRef.toMediaItem() =
|
||||||
|
MediaItem(mediaId = mediaId, kind = kind, mime = mime, size = size, filename = filename)
|
||||||
|
|
||||||
// ── message.start (open a live streaming bubble) ──────────────────────
|
// ── message.start (open a live streaming bubble) ──────────────────────
|
||||||
|
|
||||||
private fun onMessageStart(lane: String, frame: Frame) {
|
private fun onMessageStart(lane: String, frame: Frame) {
|
||||||
@@ -295,6 +335,53 @@ class ChatStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── M4: media.offer (agent produced media; pull it) ─────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attach an offered media item to its message (by [MediaOfferPayload.messageId],
|
||||||
|
* falling back to the lane's last assistant message). The item has no local
|
||||||
|
* path yet — the controller pulls it and calls [setMediaLocalPath].
|
||||||
|
*/
|
||||||
|
private fun onMediaOffer(lane: String, frame: Frame) {
|
||||||
|
val p = frame.payloadAs<MediaOfferPayload>() ?: return
|
||||||
|
updateLane(lane) { list ->
|
||||||
|
// Already attached (offer replayed)? Skip.
|
||||||
|
if (list.any { it is MessageItem && it.media.any { m -> m.mediaId == p.mediaId } }) return@updateLane list
|
||||||
|
val item = MediaItem(
|
||||||
|
mediaId = p.mediaId, kind = p.kind, mime = p.mime,
|
||||||
|
size = p.size, filename = p.filename,
|
||||||
|
)
|
||||||
|
val targetIdx = if (p.messageId != null) {
|
||||||
|
list.indexOfFirst { it.id == p.messageId }
|
||||||
|
} else {
|
||||||
|
list.indexOfLast { it is MessageItem && it.role == ROLE_ASSISTANT }
|
||||||
|
}
|
||||||
|
if (targetIdx < 0) return@updateLane list
|
||||||
|
val cur = list[targetIdx] as? MessageItem ?: return@updateLane list
|
||||||
|
list.toMutableList().also { it[targetIdx] = cur.copy(media = cur.media + item) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record a pulled file's local path for [mediaId] (all lanes). */
|
||||||
|
fun setMediaLocalPath(mediaId: String, localPath: String) {
|
||||||
|
val map = _lanes.value.toMutableMap()
|
||||||
|
var changed = false
|
||||||
|
for ((lane, list) in map) {
|
||||||
|
val updated = list.map { item ->
|
||||||
|
if (item is MessageItem) {
|
||||||
|
item.copy(media = item.media.map { m ->
|
||||||
|
if (m.mediaId == mediaId) m.copy(localPath = localPath) else m
|
||||||
|
})
|
||||||
|
} else item
|
||||||
|
}
|
||||||
|
if (updated != list) {
|
||||||
|
map[lane] = updated
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (changed) _lanes.value = map
|
||||||
|
}
|
||||||
|
|
||||||
fun clear() {
|
fun clear() {
|
||||||
_lanes.value = emptyMap()
|
_lanes.value = emptyMap()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
/** A readable local file (expect/actual; JVM impl in jvmMain). */
|
||||||
|
expect class FileSource(path: String) : AutoCloseable {
|
||||||
|
/** Total size in bytes. */
|
||||||
|
fun size(): Long
|
||||||
|
|
||||||
|
/** Read up to [buf.size] bytes into [buf]; returns bytes read or -1 at EOF. */
|
||||||
|
fun read(buf: ByteArray): Int
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Local media cache (docs/07 §7.6). Pulled media is written under the
|
||||||
|
* app-specific cache dir; LRU eviction by total size keeps it bounded
|
||||||
|
* (default 500 MB).
|
||||||
|
*/
|
||||||
|
expect class MediaCache(baseDir: String) {
|
||||||
|
/** Path for a cached media id (null when not cached). */
|
||||||
|
fun path(mediaId: String, mime: String): String?
|
||||||
|
|
||||||
|
/** Open a writer for [mediaId]; [MediaWriter.path] is the final location. */
|
||||||
|
fun openWriter(mediaId: String, mime: String): MediaWriter
|
||||||
|
|
||||||
|
/** Delete a cached file (e.g. after a failed pull). */
|
||||||
|
fun remove(mediaId: String, mime: String)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Streaming writer for a cache entry (write chunks, then close). */
|
||||||
|
expect interface MediaWriter : AutoCloseable {
|
||||||
|
/** Final file path (valid after [close]). */
|
||||||
|
val path: String
|
||||||
|
|
||||||
|
fun write(bytes: ByteArray)
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
import iris.protocol.KIND_AUDIO
|
||||||
|
import iris.protocol.KIND_DOCUMENT
|
||||||
|
import iris.protocol.KIND_IMAGE
|
||||||
|
import iris.protocol.KIND_VIDEO
|
||||||
|
|
||||||
|
/** Map a MIME type to a media kind (docs/07 §7.1). */
|
||||||
|
fun kindFromMime(mime: String): String = when {
|
||||||
|
mime.startsWith("image/") -> KIND_IMAGE
|
||||||
|
mime.startsWith("video/") -> KIND_VIDEO
|
||||||
|
mime.startsWith("audio/") -> KIND_AUDIO
|
||||||
|
else -> KIND_DOCUMENT
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best-effort file extension for a MIME type (cache file naming). */
|
||||||
|
fun extForMime(mime: String): String = when {
|
||||||
|
mime == "image/jpeg" -> ".jpg"
|
||||||
|
mime == "image/png" -> ".png"
|
||||||
|
mime == "image/webp" -> ".webp"
|
||||||
|
mime == "image/gif" -> ".gif"
|
||||||
|
mime == "image/heic" -> ".heic"
|
||||||
|
mime == "image/heif" -> ".heif"
|
||||||
|
mime == "video/mp4" -> ".mp4"
|
||||||
|
mime == "video/webm" -> ".webm"
|
||||||
|
mime == "video/quicktime" -> ".mov"
|
||||||
|
mime == "audio/mpeg" -> ".mp3"
|
||||||
|
mime == "audio/mp4" || mime == "audio/x-m4a" -> ".m4a"
|
||||||
|
mime == "audio/ogg" -> ".ogg"
|
||||||
|
mime == "audio/wav" -> ".wav"
|
||||||
|
mime == "audio/flac" -> ".flac"
|
||||||
|
mime == "audio/aac" -> ".aac"
|
||||||
|
mime == "application/pdf" -> ".pdf"
|
||||||
|
mime == "application/zip" -> ".zip"
|
||||||
|
mime == "text/plain" -> ".txt"
|
||||||
|
else -> ".bin"
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
/** Streaming SHA-256 for uploads (hash while chunking, no full-file copy). */
|
||||||
|
expect class Sha256 constructor() {
|
||||||
|
fun update(bytes: ByteArray, offset: Int = 0, length: Int = bytes.size - offset)
|
||||||
|
|
||||||
|
/** Lowercase hex digest of everything fed so far. */
|
||||||
|
fun hex(): String
|
||||||
|
}
|
||||||
@@ -1,21 +1,31 @@
|
|||||||
package iris.net
|
package iris.net
|
||||||
|
|
||||||
import iris.data.SecureStore
|
import iris.data.SecureStore
|
||||||
|
import iris.media.FileSource
|
||||||
|
import iris.media.Sha256
|
||||||
import iris.protocol.ChannelInfo
|
import iris.protocol.ChannelInfo
|
||||||
import iris.protocol.ErrorPayload
|
import iris.protocol.ErrorPayload
|
||||||
import iris.protocol.Frame
|
import iris.protocol.Frame
|
||||||
import iris.protocol.HelloAckPayload
|
import iris.protocol.HelloAckPayload
|
||||||
import iris.protocol.IrisJson
|
import iris.protocol.IrisJson
|
||||||
|
import iris.protocol.MediaPullEndPayload
|
||||||
|
import iris.protocol.MediaUploadAckPayload
|
||||||
import iris.protocol.ServerCaps
|
import iris.protocol.ServerCaps
|
||||||
import iris.protocol.TYPE_ERROR
|
import iris.protocol.TYPE_ERROR
|
||||||
import iris.protocol.TYPE_HELLO_ACK
|
import iris.protocol.TYPE_HELLO_ACK
|
||||||
|
import iris.protocol.TYPE_MEDIA_PULL_END
|
||||||
|
import iris.protocol.TYPE_MEDIA_UPLOAD_ACK
|
||||||
import iris.protocol.TYPE_PONG
|
import iris.protocol.TYPE_PONG
|
||||||
import iris.protocol.helloFrame
|
import iris.protocol.helloFrame
|
||||||
|
import iris.protocol.mediaPullFrame
|
||||||
|
import iris.protocol.mediaUploadEndFrame
|
||||||
|
import iris.protocol.mediaUploadStartFrame
|
||||||
import iris.protocol.messageSendFrame
|
import iris.protocol.messageSendFrame
|
||||||
import iris.protocol.pingFrame
|
import iris.protocol.pingFrame
|
||||||
import kotlinx.coroutines.CompletableDeferred
|
import kotlinx.coroutines.CompletableDeferred
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
|
import kotlinx.coroutines.channels.Channel
|
||||||
import kotlinx.coroutines.currentCoroutineContext
|
import kotlinx.coroutines.currentCoroutineContext
|
||||||
import kotlinx.coroutines.delay
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.isActive
|
import kotlinx.coroutines.isActive
|
||||||
@@ -26,10 +36,15 @@ import kotlinx.coroutines.flow.SharedFlow
|
|||||||
import kotlinx.coroutines.flow.StateFlow
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
import kotlinx.coroutines.flow.asSharedFlow
|
import kotlinx.coroutines.flow.asSharedFlow
|
||||||
import kotlinx.coroutines.flow.asStateFlow
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
|
import kotlinx.coroutines.sync.Mutex
|
||||||
|
import kotlinx.coroutines.sync.withLock
|
||||||
|
import kotlinx.coroutines.withTimeout
|
||||||
import kotlinx.coroutines.withTimeoutOrNull
|
import kotlinx.coroutines.withTimeoutOrNull
|
||||||
import kotlin.time.TimeMark
|
import kotlin.time.TimeMark
|
||||||
import kotlin.time.TimeSource
|
import kotlin.time.TimeSource
|
||||||
import okhttp3.OkHttpClient
|
import okhttp3.OkHttpClient
|
||||||
|
import okio.ByteString
|
||||||
|
import okio.ByteString.Companion.toByteString
|
||||||
import okhttp3.Request
|
import okhttp3.Request
|
||||||
import okhttp3.Response
|
import okhttp3.Response
|
||||||
import okhttp3.WebSocket
|
import okhttp3.WebSocket
|
||||||
@@ -75,6 +90,23 @@ class GatewayClient(
|
|||||||
private var lastLiveness: TimeMark = TimeSource.Monotonic.markNow()
|
private var lastLiveness: TimeMark = TimeSource.Monotonic.markNow()
|
||||||
private val pending = mutableMapOf<Int, CompletableDeferred<Frame>>()
|
private val pending = mutableMapOf<Int, CompletableDeferred<Frame>>()
|
||||||
|
|
||||||
|
// M4: binary frames (media upload chunks / pull stream) have no per-frame
|
||||||
|
// id, so at most one binary session is active per socket. The gateway
|
||||||
|
// allows one upload per connection; pull is request/response.
|
||||||
|
private sealed interface BinarySession {
|
||||||
|
data class Pulling(
|
||||||
|
val requestId: Int,
|
||||||
|
val chunks: Channel<ByteArray>,
|
||||||
|
val end: CompletableDeferred<Frame>,
|
||||||
|
) : BinarySession
|
||||||
|
}
|
||||||
|
|
||||||
|
private var binarySession: BinarySession? = null
|
||||||
|
|
||||||
|
// M4: only one pull may be in flight at a time (binarySession is a single
|
||||||
|
// slot). Serialize concurrent offers so their byte streams don't interleave.
|
||||||
|
private val pullMutex = Mutex()
|
||||||
|
|
||||||
// ── Lifecycle ─────────────────────────────────────────────────────────
|
// ── Lifecycle ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Start (or resume) the connect loop. Idempotent. */
|
/** Start (or resume) the connect loop. Idempotent. */
|
||||||
@@ -177,11 +209,30 @@ class GatewayClient(
|
|||||||
TYPE_PONG -> Unit
|
TYPE_PONG -> Unit
|
||||||
else -> {
|
else -> {
|
||||||
_events.tryEmit(frame)
|
_events.tryEmit(frame)
|
||||||
frame.id?.let { pending[it]?.complete(frame) }
|
frame.id?.let { id ->
|
||||||
|
pending[id]?.complete(frame)
|
||||||
|
// M4: terminal frame of a pull stream — close
|
||||||
|
// the chunk channel so the pull loop exits.
|
||||||
|
if (frame.type == TYPE_MEDIA_PULL_END) {
|
||||||
|
(binarySession as? BinarySession.Pulling)?.let {
|
||||||
|
it.chunks.close()
|
||||||
|
binarySession = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun onMessage(webSocket: WebSocket, bytes: ByteString) {
|
||||||
|
lastLiveness = TimeSource.Monotonic.markNow()
|
||||||
|
// M4: binary frames belong to the active pull stream
|
||||||
|
// (uploads are outbound; stray inbound chunks are dropped).
|
||||||
|
(binarySession as? BinarySession.Pulling)
|
||||||
|
?.chunks
|
||||||
|
?.trySend(bytes.toByteArray())
|
||||||
|
}
|
||||||
|
|
||||||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
||||||
closed.complete(Unit)
|
closed.complete(Unit)
|
||||||
}
|
}
|
||||||
@@ -215,11 +266,122 @@ class GatewayClient(
|
|||||||
|
|
||||||
// ── Outbound ──────────────────────────────────────────────────────────
|
// ── Outbound ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Send a text message (fire-and-forget; the server echoes it back). */
|
/** Send a text message (fire-and-forget; the server echoes it back).
|
||||||
fun sendMessage(chatId: String, text: String, threadId: String? = null) {
|
* M4: [mediaRefs] reference completed uploads (media.upload.ack refs). */
|
||||||
|
fun sendMessage(
|
||||||
|
chatId: String,
|
||||||
|
text: String,
|
||||||
|
threadId: String? = null,
|
||||||
|
mediaRefs: List<String> = emptyList(),
|
||||||
|
) {
|
||||||
val ws = socket ?: return
|
val ws = socket ?: return
|
||||||
val id = nextRequestId++
|
val id = nextRequestId++
|
||||||
ws.send(messageSendFrame(id, chatId, text, threadId).toWire())
|
ws.send(messageSendFrame(id, chatId, text, threadId, mediaRefs).toWire())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── M4: media upload / pull ───────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a local file as media (docs/07 §7.2): media.upload.start,
|
||||||
|
* 256 KiB binary chunks, media.upload.end {sha256}. Returns the server's
|
||||||
|
* media_ref (for message.send media_refs) on success.
|
||||||
|
*/
|
||||||
|
suspend fun uploadMedia(
|
||||||
|
path: String,
|
||||||
|
mime: String,
|
||||||
|
kind: String,
|
||||||
|
filename: String,
|
||||||
|
mediaRef: String,
|
||||||
|
): Result<String> {
|
||||||
|
val ws = socket ?: return Result.failure(IllegalStateException("not connected"))
|
||||||
|
val source = FileSource(path)
|
||||||
|
val size = source.size()
|
||||||
|
if (size <= 0) {
|
||||||
|
source.close()
|
||||||
|
return Result.failure(IllegalStateException("empty file"))
|
||||||
|
}
|
||||||
|
val id = nextRequestId++
|
||||||
|
val reply = CompletableDeferred<Frame>()
|
||||||
|
pending[id] = reply
|
||||||
|
try {
|
||||||
|
ws.send(mediaUploadStartFrame(id, mediaRef, kind, mime, filename, size).toWire())
|
||||||
|
val sha = Sha256()
|
||||||
|
source.use {
|
||||||
|
val buf = ByteArray(UPLOAD_CHUNK_BYTES)
|
||||||
|
while (true) {
|
||||||
|
val n = it.read(buf)
|
||||||
|
if (n < 0) break
|
||||||
|
if (n == 0) continue
|
||||||
|
sha.update(buf, 0, n)
|
||||||
|
ws.send(buf.copyOfRange(0, n).toByteString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ws.send(mediaUploadEndFrame(id, mediaRef, sha.hex()).toWire())
|
||||||
|
val frame = withTimeout(UPLOAD_TIMEOUT_MS) { reply.await() }
|
||||||
|
return when (frame.type) {
|
||||||
|
TYPE_MEDIA_UPLOAD_ACK -> {
|
||||||
|
val p = frame.payloadAs<MediaUploadAckPayload>()
|
||||||
|
if (p != null && p.ok) Result.success(p.mediaRef)
|
||||||
|
else Result.failure(IllegalStateException("upload rejected by server"))
|
||||||
|
}
|
||||||
|
TYPE_ERROR -> {
|
||||||
|
val e = frame.payloadAs<ErrorPayload>()
|
||||||
|
Result.failure(IllegalStateException(e?.message ?: "upload failed"))
|
||||||
|
}
|
||||||
|
else -> Result.failure(IllegalStateException("unexpected reply ${frame.type}"))
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
return Result.failure(e)
|
||||||
|
} finally {
|
||||||
|
pending.remove(id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pull offered media (docs/07 §7.3): media.pull, then binary frames until
|
||||||
|
* media.pull.end. Each chunk is handed to [onChunk] (write to cache).
|
||||||
|
*/
|
||||||
|
suspend fun pullMedia(mediaId: String, onChunk: suspend (ByteArray) -> Unit): Result<Unit> =
|
||||||
|
pullMutex.withLock {
|
||||||
|
val ws = socket ?: return@withLock Result.failure(IllegalStateException("not connected"))
|
||||||
|
val id = nextRequestId++
|
||||||
|
val chunks = Channel<ByteArray>(Channel.UNLIMITED)
|
||||||
|
val end = CompletableDeferred<Frame>()
|
||||||
|
pending[id] = end
|
||||||
|
binarySession = BinarySession.Pulling(id, chunks, end)
|
||||||
|
try {
|
||||||
|
ws.send(mediaPullFrame(id, mediaId).toWire())
|
||||||
|
val frame = withTimeout(PULL_TIMEOUT_MS) {
|
||||||
|
for (chunk in chunks) onChunk(chunk)
|
||||||
|
end.await()
|
||||||
|
}
|
||||||
|
when (frame.type) {
|
||||||
|
TYPE_MEDIA_PULL_END -> {
|
||||||
|
val p = frame.payloadAs<MediaPullEndPayload>()
|
||||||
|
if (p != null && p.ok) Result.success(Unit)
|
||||||
|
else Result.failure(IllegalStateException("pull failed"))
|
||||||
|
}
|
||||||
|
TYPE_ERROR -> {
|
||||||
|
val e = frame.payloadAs<ErrorPayload>()
|
||||||
|
Result.failure(IllegalStateException(e?.message ?: "pull failed"))
|
||||||
|
}
|
||||||
|
else -> Result.failure(IllegalStateException("unexpected reply ${frame.type}"))
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Result.failure(e)
|
||||||
|
} finally {
|
||||||
|
pending.remove(id)
|
||||||
|
chunks.cancel()
|
||||||
|
val s = binarySession
|
||||||
|
if (s is BinarySession.Pulling && s.requestId == id) binarySession = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
/** One WS binary frame carries at most this many media bytes (docs/07 §7.5). */
|
||||||
|
const val UPLOAD_CHUNK_BYTES = 256 * 1024
|
||||||
|
const val UPLOAD_TIMEOUT_MS = 120_000L
|
||||||
|
const val PULL_TIMEOUT_MS = 300_000L
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package iris.platform
|
||||||
|
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.graphics.ImageBitmap
|
||||||
|
import iris.data.MediaItem
|
||||||
|
|
||||||
|
/** A file picked by the user, staged as a local file the app can read. */
|
||||||
|
data class PickedFile(
|
||||||
|
val path: String,
|
||||||
|
val name: String,
|
||||||
|
val mime: String,
|
||||||
|
val size: Long,
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* File picker (SAF on Android, FileChooser on desktop). Invokes [onPicked]
|
||||||
|
* with the staged file, or null when the user cancels.
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
expect fun MediaFilePicker(onPicked: (PickedFile?) -> Unit)
|
||||||
|
|
||||||
|
/** Base directory for the media cache (app-specific cache dir). */
|
||||||
|
expect fun mediaCacheBaseDir(): String
|
||||||
|
|
||||||
|
/** Load an image from a local path for display. Null on failure. */
|
||||||
|
expect suspend fun loadMediaImage(path: String): ImageBitmap?
|
||||||
|
|
||||||
|
/** Inline audio/video player (ExoPlayer on Android; placeholder on desktop). */
|
||||||
|
@Composable
|
||||||
|
expect fun MediaPlayerView(media: MediaItem, modifier: Modifier)
|
||||||
|
|
||||||
|
/** Open a document with the system handler (best effort). */
|
||||||
|
expect fun openDocument(path: String, mime: String)
|
||||||
@@ -46,6 +46,14 @@ const val TYPE_TOOL_PROGRESS = "tool.progress"
|
|||||||
const val TYPE_TOOL_END = "tool.end"
|
const val TYPE_TOOL_END = "tool.end"
|
||||||
const val TYPE_COMMENTARY = "commentary"
|
const val TYPE_COMMENTARY = "commentary"
|
||||||
|
|
||||||
|
// M4 — media (upload / offer / pull)
|
||||||
|
const val TYPE_MEDIA_UPLOAD_START = "media.upload.start"
|
||||||
|
const val TYPE_MEDIA_UPLOAD_END = "media.upload.end"
|
||||||
|
const val TYPE_MEDIA_UPLOAD_ACK = "media.upload.ack"
|
||||||
|
const val TYPE_MEDIA_OFFER = "media.offer"
|
||||||
|
const val TYPE_MEDIA_PULL = "media.pull"
|
||||||
|
const val TYPE_MEDIA_PULL_END = "media.pull.end"
|
||||||
|
|
||||||
// M3 — channels / threads / search / sync
|
// M3 — channels / threads / search / sync
|
||||||
const val TYPE_CHANNEL_CREATE = "channel.create"
|
const val TYPE_CHANNEL_CREATE = "channel.create"
|
||||||
const val TYPE_CHANNEL_RENAME = "channel.rename"
|
const val TYPE_CHANNEL_RENAME = "channel.rename"
|
||||||
@@ -66,6 +74,14 @@ const val ERR_AUTH = "auth"
|
|||||||
const val ERR_NOT_FOUND = "not_found"
|
const val ERR_NOT_FOUND = "not_found"
|
||||||
const val ERR_UNSUPPORTED = "unsupported"
|
const val ERR_UNSUPPORTED = "unsupported"
|
||||||
const val ERR_INTERNAL = "internal"
|
const val ERR_INTERNAL = "internal"
|
||||||
|
const val ERR_MEDIA_TOO_LARGE = "media_too_large"
|
||||||
|
|
||||||
|
// M4 — media kinds (docs/07 §7.1)
|
||||||
|
const val KIND_IMAGE = "image"
|
||||||
|
const val KIND_AUDIO = "audio"
|
||||||
|
const val KIND_VIDEO = "video"
|
||||||
|
const val KIND_DOCUMENT = "document"
|
||||||
|
const val KIND_VOICE = "voice"
|
||||||
|
|
||||||
// ── Roles ───────────────────────────────────────────────────────────────
|
// ── Roles ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -147,8 +163,60 @@ data class MessagePayload(
|
|||||||
val model: String? = null,
|
val model: String? = null,
|
||||||
val tokens: Int? = null,
|
val tokens: Int? = null,
|
||||||
val ts: Long? = null,
|
val ts: Long? = null,
|
||||||
|
val media: List<MediaRef> = emptyList(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ── M4: media ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** A media attachment referenced by a message (or offered for pull). */
|
||||||
|
@Serializable
|
||||||
|
data class MediaRef(
|
||||||
|
@SerialName("media_id") val mediaId: String,
|
||||||
|
val kind: String,
|
||||||
|
val mime: String,
|
||||||
|
val size: Long,
|
||||||
|
val filename: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaUploadStartPayload(
|
||||||
|
@SerialName("media_ref") val mediaRef: String,
|
||||||
|
val kind: String,
|
||||||
|
val mime: String,
|
||||||
|
val filename: String,
|
||||||
|
val size: Long,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaUploadEndPayload(
|
||||||
|
@SerialName("media_ref") val mediaRef: String,
|
||||||
|
@SerialName("sha256") val sha256: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaUploadAckPayload(
|
||||||
|
val ok: Boolean,
|
||||||
|
@SerialName("media_ref") val mediaRef: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaOfferPayload(
|
||||||
|
@SerialName("media_id") val mediaId: String,
|
||||||
|
val kind: String,
|
||||||
|
val mime: String,
|
||||||
|
val size: Long,
|
||||||
|
val filename: String,
|
||||||
|
@SerialName("message_id") val messageId: String? = null,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaPullPayload(
|
||||||
|
@SerialName("media_id") val mediaId: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class MediaPullEndPayload(val ok: Boolean)
|
||||||
|
|
||||||
// ── M2: streaming frames (server -> app) ────────────────────────────────
|
// ── M2: streaming frames (server -> app) ────────────────────────────────
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
@@ -292,7 +360,13 @@ fun helloFrame(token: String, deviceId: String, deviceName: String): Frame =
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
fun messageSendFrame(id: Int, chatId: String, text: String, threadId: String? = null): Frame =
|
fun messageSendFrame(
|
||||||
|
id: Int,
|
||||||
|
chatId: String,
|
||||||
|
text: String,
|
||||||
|
threadId: String? = null,
|
||||||
|
mediaRefs: List<String> = emptyList(),
|
||||||
|
): Frame =
|
||||||
Frame(
|
Frame(
|
||||||
id = id,
|
id = id,
|
||||||
type = TYPE_MESSAGE_SEND,
|
type = TYPE_MESSAGE_SEND,
|
||||||
@@ -300,7 +374,7 @@ fun messageSendFrame(id: Int, chatId: String, text: String, threadId: String? =
|
|||||||
threadId = threadId,
|
threadId = threadId,
|
||||||
payload = IrisJson.instance.encodeToJsonElement(
|
payload = IrisJson.instance.encodeToJsonElement(
|
||||||
MessageSendPayload.serializer(),
|
MessageSendPayload.serializer(),
|
||||||
MessageSendPayload(text = text),
|
MessageSendPayload(text = text, mediaRefs = mediaRefs),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -365,4 +439,43 @@ fun syncFrame(id: Int, cursor: Long): Frame =
|
|||||||
SyncPayload.serializer(),
|
SyncPayload.serializer(),
|
||||||
SyncPayload(cursor = cursor),
|
SyncPayload(cursor = cursor),
|
||||||
),
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── M4 frame builders ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fun mediaUploadStartFrame(
|
||||||
|
id: Int,
|
||||||
|
mediaRef: String,
|
||||||
|
kind: String,
|
||||||
|
mime: String,
|
||||||
|
filename: String,
|
||||||
|
size: Long,
|
||||||
|
): Frame =
|
||||||
|
Frame(
|
||||||
|
id = id,
|
||||||
|
type = TYPE_MEDIA_UPLOAD_START,
|
||||||
|
payload = IrisJson.instance.encodeToJsonElement(
|
||||||
|
MediaUploadStartPayload.serializer(),
|
||||||
|
MediaUploadStartPayload(mediaRef, kind, mime, filename, size),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
fun mediaUploadEndFrame(id: Int, mediaRef: String, sha256: String): Frame =
|
||||||
|
Frame(
|
||||||
|
id = id,
|
||||||
|
type = TYPE_MEDIA_UPLOAD_END,
|
||||||
|
payload = IrisJson.instance.encodeToJsonElement(
|
||||||
|
MediaUploadEndPayload.serializer(),
|
||||||
|
MediaUploadEndPayload(mediaRef, sha256),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
fun mediaPullFrame(id: Int, mediaId: String): Frame =
|
||||||
|
Frame(
|
||||||
|
id = id,
|
||||||
|
type = TYPE_MEDIA_PULL,
|
||||||
|
payload = IrisJson.instance.encodeToJsonElement(
|
||||||
|
MediaPullPayload.serializer(),
|
||||||
|
MediaPullPayload(mediaId),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
@@ -2,8 +2,14 @@ package iris.state
|
|||||||
|
|
||||||
import iris.data.ChatStore
|
import iris.data.ChatStore
|
||||||
import iris.data.ChannelStore
|
import iris.data.ChannelStore
|
||||||
|
import iris.data.MediaItem
|
||||||
import iris.data.SecureStore
|
import iris.data.SecureStore
|
||||||
|
import iris.media.MediaCache
|
||||||
|
import iris.media.kindFromMime
|
||||||
import iris.net.GatewayClient
|
import iris.net.GatewayClient
|
||||||
|
import iris.platform.PickedFile
|
||||||
|
import iris.platform.mediaCacheBaseDir
|
||||||
|
import iris.protocol.MediaOfferPayload
|
||||||
import iris.protocol.SearchHit
|
import iris.protocol.SearchHit
|
||||||
import iris.protocol.SearchResultsPayload
|
import iris.protocol.SearchResultsPayload
|
||||||
import iris.protocol.SyncDonePayload
|
import iris.protocol.SyncDonePayload
|
||||||
@@ -12,6 +18,7 @@ import iris.protocol.TYPE_CHANNEL_DELETED
|
|||||||
import iris.protocol.TYPE_CHANNEL_LIST
|
import iris.protocol.TYPE_CHANNEL_LIST
|
||||||
import iris.protocol.TYPE_CHANNEL_RENAMED
|
import iris.protocol.TYPE_CHANNEL_RENAMED
|
||||||
import iris.protocol.TYPE_COMMENTARY
|
import iris.protocol.TYPE_COMMENTARY
|
||||||
|
import iris.protocol.TYPE_MEDIA_OFFER
|
||||||
import iris.protocol.TYPE_MESSAGE
|
import iris.protocol.TYPE_MESSAGE
|
||||||
import iris.protocol.TYPE_MESSAGE_START
|
import iris.protocol.TYPE_MESSAGE_START
|
||||||
import iris.protocol.TYPE_MESSAGE_STOP
|
import iris.protocol.TYPE_MESSAGE_STOP
|
||||||
@@ -37,6 +44,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
|
|||||||
import kotlinx.coroutines.flow.StateFlow
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
import kotlinx.coroutines.flow.asStateFlow
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* App-level controller (M3): owns the GatewayClient + ChatStore + ChannelStore,
|
* App-level controller (M3): owns the GatewayClient + ChatStore + ChannelStore,
|
||||||
@@ -88,6 +96,24 @@ class IrisController(
|
|||||||
private val _lastQuery = MutableStateFlow("")
|
private val _lastQuery = MutableStateFlow("")
|
||||||
val lastQuery: StateFlow<String> = _lastQuery.asStateFlow()
|
val lastQuery: StateFlow<String> = _lastQuery.asStateFlow()
|
||||||
|
|
||||||
|
// ── M4: media ─────────────────────────────────────────────────────────
|
||||||
|
private val mediaCache = MediaCache(mediaCacheBaseDir())
|
||||||
|
|
||||||
|
/** A composer attachment: picked file being uploaded (or uploaded). */
|
||||||
|
data class PendingAttachment(
|
||||||
|
val filename: String,
|
||||||
|
val mime: String,
|
||||||
|
val size: Long,
|
||||||
|
val kind: String,
|
||||||
|
val path: String? = null,
|
||||||
|
val uploading: Boolean = true,
|
||||||
|
val mediaRef: String? = null,
|
||||||
|
val error: String? = null,
|
||||||
|
)
|
||||||
|
|
||||||
|
private val _attachments = MutableStateFlow<List<PendingAttachment>>(emptyList())
|
||||||
|
val attachments: StateFlow<List<PendingAttachment>> = _attachments.asStateFlow()
|
||||||
|
|
||||||
init {
|
init {
|
||||||
scope.launch {
|
scope.launch {
|
||||||
client.events.collect { frame ->
|
client.events.collect { frame ->
|
||||||
@@ -99,7 +125,14 @@ class IrisController(
|
|||||||
TYPE_TOOL_START,
|
TYPE_TOOL_START,
|
||||||
TYPE_TOOL_PROGRESS,
|
TYPE_TOOL_PROGRESS,
|
||||||
TYPE_TOOL_END,
|
TYPE_TOOL_END,
|
||||||
TYPE_COMMENTARY -> chat.onFrame(frame)
|
TYPE_COMMENTARY,
|
||||||
|
TYPE_MEDIA_OFFER -> {
|
||||||
|
chat.onFrame(frame)
|
||||||
|
// M4: pull offered media into the local cache.
|
||||||
|
if (frame.type == TYPE_MEDIA_OFFER) {
|
||||||
|
frame.payloadAs<MediaOfferPayload>()?.let { pullMedia(it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
TYPE_CHANNEL_CREATED,
|
TYPE_CHANNEL_CREATED,
|
||||||
TYPE_CHANNEL_RENAMED,
|
TYPE_CHANNEL_RENAMED,
|
||||||
TYPE_CHANNEL_DELETED,
|
TYPE_CHANNEL_DELETED,
|
||||||
@@ -202,14 +235,76 @@ class IrisController(
|
|||||||
client.sendFrame(syncFrame(0, cursor))
|
client.sendFrame(syncFrame(0, cursor))
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Optimistic send: show immediately in the current lane, then hand to the gateway. */
|
/** Optimistic send: show immediately in the current lane, then hand to the gateway.
|
||||||
fun send(text: String) {
|
* M4: [attachments] (uploaded) are attached via media_refs. */
|
||||||
|
fun send(text: String, attachments: List<PendingAttachment> = emptyList()) {
|
||||||
val trimmed = text.trim()
|
val trimmed = text.trim()
|
||||||
if (trimmed.isEmpty()) return
|
val refs = attachments.mapNotNull { it.mediaRef }
|
||||||
|
if (trimmed.isEmpty() && refs.isEmpty()) return
|
||||||
val lane = chat.currentLane.value
|
val lane = chat.currentLane.value
|
||||||
val (chatId, threadId) = chat.parseLane(lane)
|
val (chatId, threadId) = chat.parseLane(lane)
|
||||||
chat.addPending(trimmed, lane)
|
val media = attachments.map {
|
||||||
client.sendMessage(chatId, trimmed, threadId)
|
MediaItem(
|
||||||
|
mediaId = it.mediaRef ?: it.filename,
|
||||||
|
kind = it.kind, mime = it.mime, size = it.size, filename = it.filename,
|
||||||
|
localPath = it.path,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
chat.addPending(trimmed, lane, media)
|
||||||
|
client.sendMessage(chatId, trimmed, threadId, refs)
|
||||||
|
_attachments.value = emptyList()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── M4: attachments (pick -> upload -> send) ──────────────────────────
|
||||||
|
|
||||||
|
/** Stage a picked file: upload it, then keep it as a pending attachment. */
|
||||||
|
fun attachFile(picked: PickedFile) {
|
||||||
|
val kind = kindFromMime(picked.mime)
|
||||||
|
val placeholder = PendingAttachment(
|
||||||
|
filename = picked.name, mime = picked.mime, size = picked.size, kind = kind,
|
||||||
|
path = picked.path,
|
||||||
|
)
|
||||||
|
_attachments.value = _attachments.value + placeholder
|
||||||
|
scope.launch {
|
||||||
|
val mediaRef = "up_${Random.nextLong(1_000_000_000L, 9_999_999_999L)}"
|
||||||
|
val result = client.uploadMedia(
|
||||||
|
path = picked.path,
|
||||||
|
mime = picked.mime,
|
||||||
|
kind = kind,
|
||||||
|
filename = picked.name,
|
||||||
|
mediaRef = mediaRef,
|
||||||
|
)
|
||||||
|
_attachments.value = _attachments.value.map {
|
||||||
|
if (it.filename == picked.name && it.uploading) {
|
||||||
|
result.fold(
|
||||||
|
{ ref -> it.copy(uploading = false, mediaRef = ref) },
|
||||||
|
{ e -> it.copy(uploading = false, error = e.message) },
|
||||||
|
)
|
||||||
|
} else it
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun removeAttachment(filename: String) {
|
||||||
|
_attachments.value = _attachments.value.filterNot { it.filename == filename }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pull offered media into the local cache and record the path. */
|
||||||
|
private fun pullMedia(offer: MediaOfferPayload) {
|
||||||
|
scope.launch {
|
||||||
|
// Already cached? Skip the pull.
|
||||||
|
mediaCache.path(offer.mediaId, offer.mime)?.let {
|
||||||
|
chat.setMediaLocalPath(offer.mediaId, it)
|
||||||
|
return@launch
|
||||||
|
}
|
||||||
|
val writer = mediaCache.openWriter(offer.mediaId, offer.mime)
|
||||||
|
val result = client.pullMedia(offer.mediaId) { chunk -> writer.write(chunk) }
|
||||||
|
writer.close()
|
||||||
|
result.fold(
|
||||||
|
{ chat.setMediaLocalPath(offer.mediaId, writer.path) },
|
||||||
|
{ mediaCache.remove(offer.mediaId, offer.mime) },
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Connect-screen action: real hello test, then save + (re)connect. */
|
/** Connect-screen action: real hello test, then save + (re)connect. */
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ import androidx.compose.foundation.layout.heightIn
|
|||||||
import androidx.compose.foundation.layout.padding
|
import androidx.compose.foundation.layout.padding
|
||||||
import androidx.compose.foundation.layout.size
|
import androidx.compose.foundation.layout.size
|
||||||
import androidx.compose.foundation.layout.width
|
import androidx.compose.foundation.layout.width
|
||||||
|
import androidx.compose.foundation.Image
|
||||||
|
import androidx.compose.foundation.layout.fillMaxHeight
|
||||||
import androidx.compose.foundation.lazy.LazyColumn
|
import androidx.compose.foundation.lazy.LazyColumn
|
||||||
import androidx.compose.foundation.lazy.items
|
import androidx.compose.foundation.lazy.items
|
||||||
import androidx.compose.foundation.lazy.rememberLazyListState
|
import androidx.compose.foundation.lazy.rememberLazyListState
|
||||||
@@ -47,7 +49,10 @@ import androidx.compose.ui.Alignment
|
|||||||
import androidx.compose.ui.Modifier
|
import androidx.compose.ui.Modifier
|
||||||
import androidx.compose.ui.draw.clip
|
import androidx.compose.ui.draw.clip
|
||||||
import androidx.compose.ui.graphics.Color
|
import androidx.compose.ui.graphics.Color
|
||||||
|
import androidx.compose.ui.graphics.ImageBitmap
|
||||||
|
import androidx.compose.ui.layout.ContentScale
|
||||||
import androidx.compose.ui.platform.LocalClipboardManager
|
import androidx.compose.ui.platform.LocalClipboardManager
|
||||||
|
import androidx.compose.ui.platform.LocalFocusManager
|
||||||
import androidx.compose.ui.text.AnnotatedString
|
import androidx.compose.ui.text.AnnotatedString
|
||||||
import androidx.compose.ui.text.font.FontFamily
|
import androidx.compose.ui.text.font.FontFamily
|
||||||
import androidx.compose.ui.text.font.FontWeight
|
import androidx.compose.ui.text.font.FontWeight
|
||||||
@@ -56,10 +61,17 @@ import androidx.compose.ui.unit.dp
|
|||||||
import androidx.compose.ui.unit.sp
|
import androidx.compose.ui.unit.sp
|
||||||
import iris.data.ChannelStore
|
import iris.data.ChannelStore
|
||||||
import iris.data.ChatItem
|
import iris.data.ChatItem
|
||||||
|
import iris.data.MediaItem
|
||||||
import iris.data.MessageItem
|
import iris.data.MessageItem
|
||||||
import iris.data.ToolItem
|
import iris.data.ToolItem
|
||||||
import iris.net.GatewayClient
|
import iris.net.GatewayClient
|
||||||
|
import iris.platform.MediaFilePicker
|
||||||
|
import iris.platform.MediaPlayerView
|
||||||
|
import iris.platform.loadMediaImage
|
||||||
|
import iris.platform.openDocument
|
||||||
import iris.protocol.ChannelInfo
|
import iris.protocol.ChannelInfo
|
||||||
|
import iris.protocol.KIND_DOCUMENT
|
||||||
|
import iris.protocol.KIND_IMAGE
|
||||||
import iris.protocol.ROLE_USER
|
import iris.protocol.ROLE_USER
|
||||||
import iris.protocol.SearchHit
|
import iris.protocol.SearchHit
|
||||||
import iris.state.IrisController
|
import iris.state.IrisController
|
||||||
@@ -91,14 +103,20 @@ fun ChatScreen(controller: IrisController) {
|
|||||||
var showSearch by remember { mutableStateOf(false) }
|
var showSearch by remember { mutableStateOf(false) }
|
||||||
var showNewChannel by remember { mutableStateOf(false) }
|
var showNewChannel by remember { mutableStateOf(false) }
|
||||||
var showNewThread by remember { mutableStateOf(false) }
|
var showNewThread by remember { mutableStateOf(false) }
|
||||||
|
var showPicker by remember { mutableStateOf(false) }
|
||||||
|
val attachments by controller.attachments.collectAsState()
|
||||||
val drawerState = rememberDrawerState(DrawerValue.Closed)
|
val drawerState = rememberDrawerState(DrawerValue.Closed)
|
||||||
val drawerScope = rememberCoroutineScope()
|
val drawerScope = rememberCoroutineScope()
|
||||||
|
val focusManager = LocalFocusManager.current
|
||||||
|
|
||||||
fun doSend() {
|
fun doSend() {
|
||||||
if (input.isBlank()) return
|
val ready = attachments.filter { it.mediaRef != null && it.error == null }
|
||||||
|
if (input.isBlank() && ready.isEmpty()) return
|
||||||
val text = input
|
val text = input
|
||||||
input = ""
|
input = ""
|
||||||
controller.send(text)
|
controller.send(text, ready)
|
||||||
|
// Best practice: dismiss the keyboard after sending.
|
||||||
|
focusManager.clearFocus(force = true)
|
||||||
}
|
}
|
||||||
|
|
||||||
LaunchedEffect(items.size, typing) {
|
LaunchedEffect(items.size, typing) {
|
||||||
@@ -206,6 +224,21 @@ fun ChatScreen(controller: IrisController) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pending attachments (M4)
|
||||||
|
if (attachments.isNotEmpty()) {
|
||||||
|
Row(
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.horizontalScroll(rememberScrollState())
|
||||||
|
.padding(horizontal = 12.dp, vertical = 4.dp),
|
||||||
|
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||||
|
) {
|
||||||
|
attachments.forEach { att ->
|
||||||
|
AttachmentChip(att, onRemove = { controller.removeAttachment(att.filename) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Composer
|
// Composer
|
||||||
Row(
|
Row(
|
||||||
modifier = Modifier
|
modifier = Modifier
|
||||||
@@ -213,6 +246,9 @@ fun ChatScreen(controller: IrisController) {
|
|||||||
.padding(12.dp),
|
.padding(12.dp),
|
||||||
verticalAlignment = Alignment.Bottom,
|
verticalAlignment = Alignment.Bottom,
|
||||||
) {
|
) {
|
||||||
|
IconButton(onClick = { showPicker = true }) {
|
||||||
|
Text("📎", fontSize = 18.sp)
|
||||||
|
}
|
||||||
OutlinedTextField(
|
OutlinedTextField(
|
||||||
value = input,
|
value = input,
|
||||||
onValueChange = { input = it },
|
onValueChange = { input = it },
|
||||||
@@ -226,7 +262,7 @@ fun ChatScreen(controller: IrisController) {
|
|||||||
Spacer(modifier = Modifier.width(8.dp))
|
Spacer(modifier = Modifier.width(8.dp))
|
||||||
Button(
|
Button(
|
||||||
onClick = { doSend() },
|
onClick = { doSend() },
|
||||||
enabled = input.isNotBlank(),
|
enabled = input.isNotBlank() || attachments.any { it.mediaRef != null && it.error == null },
|
||||||
) {
|
) {
|
||||||
Text("Send")
|
Text("Send")
|
||||||
}
|
}
|
||||||
@@ -234,6 +270,16 @@ fun ChatScreen(controller: IrisController) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// File picker overlay (M4)
|
||||||
|
if (showPicker) {
|
||||||
|
Box(modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.5f))) {
|
||||||
|
MediaFilePicker(onPicked = { picked ->
|
||||||
|
showPicker = false
|
||||||
|
if (picked != null) controller.attachFile(picked)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Search overlay
|
// Search overlay
|
||||||
if (showSearch) {
|
if (showSearch) {
|
||||||
SearchOverlay(
|
SearchOverlay(
|
||||||
@@ -520,6 +566,13 @@ private fun MessageBubble(msg: MessageItem) {
|
|||||||
fontSize = if (isCommentary) 13.sp else 15.sp,
|
fontSize = if (isCommentary) 13.sp else 15.sp,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
// M4: media attachments (image / player / document chip).
|
||||||
|
if (msg.media.isNotEmpty()) {
|
||||||
|
Spacer(modifier = Modifier.height(6.dp))
|
||||||
|
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||||
|
msg.media.forEach { m -> MediaAttachment(m) }
|
||||||
|
}
|
||||||
|
}
|
||||||
if (msg.pending) {
|
if (msg.pending) {
|
||||||
Text("sending…", color = textColor.copy(alpha = 0.6f), fontSize = 10.sp)
|
Text("sending…", color = textColor.copy(alpha = 0.6f), fontSize = 10.sp)
|
||||||
}
|
}
|
||||||
@@ -542,6 +595,135 @@ private fun MessageBubble(msg: MessageItem) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* M4: render one media attachment in a bubble.
|
||||||
|
* - image: tappable thumbnail (full-screen viewer)
|
||||||
|
* - audio/video: inline player (ExoPlayer on Android)
|
||||||
|
* - document: filename chip (tap to open with system handler)
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
private fun MediaAttachment(media: MediaItem) {
|
||||||
|
when (media.kind) {
|
||||||
|
KIND_IMAGE -> MediaImage(media)
|
||||||
|
KIND_DOCUMENT -> MediaDocChip(media)
|
||||||
|
else -> MediaPlayerView(media, Modifier.fillMaxWidth())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun MediaImage(media: MediaItem) {
|
||||||
|
var bitmap by remember(media.localPath) { mutableStateOf<ImageBitmap?>(null) }
|
||||||
|
var loading by remember(media.localPath) { mutableStateOf(media.localPath != null) }
|
||||||
|
var showFull by remember { mutableStateOf(false) }
|
||||||
|
|
||||||
|
LaunchedEffect(media.localPath) {
|
||||||
|
val path = media.localPath
|
||||||
|
if (path != null) {
|
||||||
|
bitmap = loadMediaImage(path)
|
||||||
|
loading = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (loading || bitmap == null) {
|
||||||
|
Box(
|
||||||
|
modifier = Modifier
|
||||||
|
.size(120.dp)
|
||||||
|
.clip(RoundedCornerShape(10.dp))
|
||||||
|
.background(Color(0xFF20242E)),
|
||||||
|
contentAlignment = Alignment.Center,
|
||||||
|
) {
|
||||||
|
if (media.localPath == null) Text("downloading…", fontSize = 11.sp)
|
||||||
|
else CircularProgressIndicator(modifier = Modifier.size(20.dp), strokeWidth = 2.dp)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
val bmp = bitmap!!
|
||||||
|
val aspect = bmp.width.toFloat() / bmp.height.toFloat()
|
||||||
|
// Preserve the image's aspect ratio, capping the dominant dimension at
|
||||||
|
// 260.dp so a chat image never dominates the screen (the other dimension
|
||||||
|
// follows the ratio).
|
||||||
|
val (imgW, imgH) = if (aspect >= 1f) 260.dp to (260 / aspect).dp
|
||||||
|
else (260 * aspect).dp to 260.dp
|
||||||
|
Image(
|
||||||
|
bitmap = bmp,
|
||||||
|
contentDescription = media.filename,
|
||||||
|
contentScale = ContentScale.FillBounds,
|
||||||
|
modifier = Modifier
|
||||||
|
.width(imgW)
|
||||||
|
.height(imgH)
|
||||||
|
.clip(RoundedCornerShape(10.dp))
|
||||||
|
.clickable { showFull = true },
|
||||||
|
)
|
||||||
|
if (showFull) {
|
||||||
|
Box(
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxSize()
|
||||||
|
.background(Color.Black)
|
||||||
|
.clickable { showFull = false },
|
||||||
|
contentAlignment = Alignment.Center,
|
||||||
|
) {
|
||||||
|
Image(
|
||||||
|
bitmap = bitmap!!,
|
||||||
|
contentDescription = media.filename,
|
||||||
|
contentScale = ContentScale.Fit,
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.fillMaxHeight(0.9f),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun MediaDocChip(media: MediaItem) {
|
||||||
|
Row(
|
||||||
|
modifier = Modifier
|
||||||
|
.clip(RoundedCornerShape(10.dp))
|
||||||
|
.background(Color(0xFF20242E))
|
||||||
|
.clickable {
|
||||||
|
media.localPath?.let { openDocument(it, media.mime) }
|
||||||
|
}
|
||||||
|
.padding(horizontal = 10.dp, vertical = 8.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
) {
|
||||||
|
Text("📄", fontSize = 16.sp)
|
||||||
|
Spacer(modifier = Modifier.width(8.dp))
|
||||||
|
Column {
|
||||||
|
Text(media.filename, fontSize = 13.sp, maxLines = 1)
|
||||||
|
Text(fmtSize(media.size), fontSize = 10.sp, color = Color(0xFF8A93A6))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun fmtSize(bytes: Long): String = when {
|
||||||
|
bytes >= 1_048_576 -> "%.1f MB".format(bytes / 1_048_576.0)
|
||||||
|
bytes >= 1024 -> "%.0f KB".format(bytes / 1024.0)
|
||||||
|
else -> "$bytes B"
|
||||||
|
}
|
||||||
|
|
||||||
|
/** M4: composer attachment chip (uploading / ready / error). */
|
||||||
|
@Composable
|
||||||
|
private fun AttachmentChip(att: IrisController.PendingAttachment, onRemove: () -> Unit) {
|
||||||
|
val (icon, label) = when {
|
||||||
|
att.error != null -> "⚠️" to "${att.filename} — ${att.error}"
|
||||||
|
att.uploading -> "⏳" to "${att.filename} (${fmtSize(att.size)})"
|
||||||
|
else -> "📎" to "${att.filename} (${fmtSize(att.size)})"
|
||||||
|
}
|
||||||
|
Row(
|
||||||
|
modifier = Modifier
|
||||||
|
.clip(RoundedCornerShape(12.dp))
|
||||||
|
.background(Color(0xFF2A2E3B))
|
||||||
|
.padding(horizontal = 8.dp, vertical = 5.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
) {
|
||||||
|
Text(icon, fontSize = 12.sp)
|
||||||
|
Spacer(modifier = Modifier.width(4.dp))
|
||||||
|
Text(label, fontSize = 12.sp, color = Color(0xFFC7CCD8), maxLines = 1)
|
||||||
|
Spacer(modifier = Modifier.width(4.dp))
|
||||||
|
TextButton(onClick = onRemove) { Text("✕", fontSize = 11.sp) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collapsible reasoning panel (M2): header "💭 Reasoning", monospace body,
|
* Collapsible reasoning panel (M2): header "💭 Reasoning", monospace body,
|
||||||
* copy button. Collapsed by default when long; tap to toggle.
|
* copy button. Collapsed by default when long; tap to toggle.
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package iris.platform
|
||||||
|
|
||||||
|
import androidx.compose.foundation.layout.Box
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.Spacer
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.foundation.layout.width
|
||||||
|
import androidx.compose.material3.Button
|
||||||
|
import androidx.compose.material3.OutlinedTextField
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.mutableStateOf
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.runtime.setValue
|
||||||
|
import androidx.compose.ui.Alignment
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.graphics.ImageBitmap
|
||||||
|
import androidx.compose.ui.graphics.asImageBitmap
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
import androidx.compose.ui.unit.sp
|
||||||
|
import iris.data.MediaItem
|
||||||
|
import java.io.File
|
||||||
|
import javax.imageio.ImageIO
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
actual fun MediaFilePicker(onPicked: (PickedFile?) -> Unit) {
|
||||||
|
var path by remember { mutableStateOf("") }
|
||||||
|
Column(
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.padding(16.dp),
|
||||||
|
) {
|
||||||
|
Text("File path (desktop):", fontSize = 13.sp)
|
||||||
|
Spacer(modifier = Modifier.width(0.dp))
|
||||||
|
OutlinedTextField(
|
||||||
|
value = path,
|
||||||
|
onValueChange = { path = it },
|
||||||
|
modifier = Modifier.fillMaxWidth(),
|
||||||
|
placeholder = { Text("/path/to/file") },
|
||||||
|
)
|
||||||
|
Spacer(modifier = Modifier.width(0.dp))
|
||||||
|
Row {
|
||||||
|
Button(onClick = {
|
||||||
|
val f = File(path.trim())
|
||||||
|
if (f.exists() && f.isFile) {
|
||||||
|
onPicked(
|
||||||
|
PickedFile(
|
||||||
|
path = f.absolutePath,
|
||||||
|
name = f.name,
|
||||||
|
mime = guessMime(f.name),
|
||||||
|
size = f.length(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
onPicked(null)
|
||||||
|
}
|
||||||
|
}, enabled = path.isNotBlank()) {
|
||||||
|
Text("Attach")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun guessMime(name: String): String {
|
||||||
|
val ext = name.substringAfterLast('.', "").lowercase()
|
||||||
|
return when (ext) {
|
||||||
|
"jpg", "jpeg" -> "image/jpeg"
|
||||||
|
"png" -> "image/png"
|
||||||
|
"webp" -> "image/webp"
|
||||||
|
"gif" -> "image/gif"
|
||||||
|
"mp4" -> "video/mp4"
|
||||||
|
"webm" -> "video/webm"
|
||||||
|
"mov" -> "video/quicktime"
|
||||||
|
"mp3" -> "audio/mpeg"
|
||||||
|
"m4a", "aac" -> "audio/mp4"
|
||||||
|
"ogg", "opus" -> "audio/ogg"
|
||||||
|
"wav" -> "audio/wav"
|
||||||
|
"pdf" -> "application/pdf"
|
||||||
|
"txt" -> "text/plain"
|
||||||
|
else -> "application/octet-stream"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun mediaCacheBaseDir(): String {
|
||||||
|
val dir = File(System.getProperty("user.home"), ".iris/cache")
|
||||||
|
dir.mkdirs()
|
||||||
|
return dir.absolutePath
|
||||||
|
}
|
||||||
|
|
||||||
|
actual suspend fun loadMediaImage(path: String): ImageBitmap? {
|
||||||
|
// Desktop image decoding via BufferedImage->ImageBitmap is a later
|
||||||
|
// milestone; return null so bubbles show the "loading" placeholder.
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
actual fun MediaPlayerView(media: MediaItem, modifier: Modifier) {
|
||||||
|
// Desktop playback backend (libmpv/WebView) is a later milestone;
|
||||||
|
// show a chip with the file info for now.
|
||||||
|
Box(modifier = modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
|
||||||
|
Text(
|
||||||
|
"${media.kind}: ${media.filename} (${media.size} B) — playback not available on desktop yet",
|
||||||
|
fontSize = 12.sp,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun openDocument(path: String, mime: String) {
|
||||||
|
// Desktop open-with is a later milestone; no-op for now.
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
|
||||||
|
actual class FileSource actual constructor(path: String) : AutoCloseable {
|
||||||
|
private val file = File(path)
|
||||||
|
private val input = file.inputStream()
|
||||||
|
|
||||||
|
actual fun size(): Long = file.length()
|
||||||
|
|
||||||
|
actual fun read(buf: ByteArray): Int = input.read(buf)
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
input.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileOutputStream
|
||||||
|
|
||||||
|
actual interface MediaWriter : AutoCloseable {
|
||||||
|
actual val path: String
|
||||||
|
actual fun write(bytes: ByteArray)
|
||||||
|
}
|
||||||
|
|
||||||
|
actual class MediaCache actual constructor(baseDir: String) {
|
||||||
|
private val maxBytes: Long = 500L * 1024 * 1024
|
||||||
|
private val mediaDir: File = File(baseDir, "media").apply { mkdirs() }
|
||||||
|
|
||||||
|
actual fun path(mediaId: String, mime: String): String? {
|
||||||
|
val f = File(mediaDir, "$mediaId${extForMime(mime)}")
|
||||||
|
return if (f.exists()) f.absolutePath else null
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun openWriter(mediaId: String, mime: String): MediaWriter =
|
||||||
|
JvmMediaWriter(File(mediaDir, "$mediaId${extForMime(mime)}"), this)
|
||||||
|
|
||||||
|
actual fun remove(mediaId: String, mime: String) {
|
||||||
|
path(mediaId, mime)?.let { File(it).delete() }
|
||||||
|
}
|
||||||
|
|
||||||
|
internal fun evict() {
|
||||||
|
val files = mediaDir.listFiles()?.filter { !it.name.endsWith(".part") } ?: return
|
||||||
|
var total = files.sumOf { it.length() }
|
||||||
|
if (total <= maxBytes) return
|
||||||
|
files.sortedBy { it.lastModified() }.forEach { f ->
|
||||||
|
if (total <= maxBytes) return
|
||||||
|
total -= f.length()
|
||||||
|
f.delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class JvmMediaWriter(private val target: File, private val cache: MediaCache) : MediaWriter {
|
||||||
|
private val out: FileOutputStream = FileOutputStream(File(target.parentFile, "${target.name}.part"))
|
||||||
|
|
||||||
|
override val path: String get() = target.absolutePath
|
||||||
|
|
||||||
|
override fun write(bytes: ByteArray) {
|
||||||
|
out.write(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
out.close()
|
||||||
|
val part = File(target.parentFile, "${target.name}.part")
|
||||||
|
if (target.exists()) target.delete()
|
||||||
|
if (!part.renameTo(target)) {
|
||||||
|
// Rare cross-device rename failure: fall back to copy.
|
||||||
|
part.copyTo(target, overwrite = true)
|
||||||
|
part.delete()
|
||||||
|
}
|
||||||
|
cache.evict()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package iris.media
|
||||||
|
|
||||||
|
import java.security.MessageDigest
|
||||||
|
|
||||||
|
actual class Sha256 actual constructor() {
|
||||||
|
private val digest = MessageDigest.getInstance("SHA-256")
|
||||||
|
|
||||||
|
actual fun update(bytes: ByteArray, offset: Int, length: Int) {
|
||||||
|
digest.update(bytes, offset, length)
|
||||||
|
}
|
||||||
|
|
||||||
|
actual fun hex(): String {
|
||||||
|
val bytes = digest.digest()
|
||||||
|
return buildString(bytes.size * 2) {
|
||||||
|
for (b in bytes) append("%02x".format(b))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -227,6 +227,13 @@ See `07-media.md`.
|
|||||||
{"type":"media.upload.end","id":11,"payload":{"media_ref":"mu_1","sha256":"…"}}
|
{"type":"media.upload.end","id":11,"payload":{"media_ref":"mu_1","sha256":"…"}}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `media.upload.ack`
|
||||||
|
Server → App response to `media.upload.end`: the ref is cached and may now be
|
||||||
|
referenced in a `message.send` `media_refs`. Failures use `error` frames instead.
|
||||||
|
```json
|
||||||
|
{"type":"media.upload.ack","id":11,"payload":{"ok":true,"media_ref":"mu_1"}}
|
||||||
|
```
|
||||||
|
|
||||||
### `media.pull`
|
### `media.pull`
|
||||||
Request agent-sent media bytes.
|
Request agent-sent media bytes.
|
||||||
```json
|
```json
|
||||||
|
|||||||
+4
-3
@@ -29,9 +29,10 @@ receipt (don't trust the client) using hermes helpers
|
|||||||
- image → `cache_image_from_bytes`
|
- image → `cache_image_from_bytes`
|
||||||
- audio/voice → `cache_audio_from_bytes`
|
- audio/voice → `cache_audio_from_bytes`
|
||||||
- video → `cache_video_from_bytes`
|
- video → `cache_video_from_bytes`
|
||||||
- document → `cache_document_from_bytes`
|
- document → `cache_document_from_bytes`
|
||||||
→ returns a local path.
|
→ returns a local path.
|
||||||
6. The path is attached to the next `message.send` via `media_refs`, becoming
|
5b. Plugin replies `media.upload.ack {ok, media_ref}` (failures use `error`).
|
||||||
|
6. The path is attached to the next `message.send` via `media_refs`, becoming
|
||||||
`MessageEvent.media_urls` + `media_types`
|
`MessageEvent.media_urls` + `media_types`
|
||||||
(`gateway/platforms/base.py:2337`). The agent's vision/audio tools can then
|
(`gateway/platforms/base.py:2337`). The agent's vision/audio tools can then
|
||||||
read the file.
|
read the file.
|
||||||
|
|||||||
+17
-3
@@ -90,17 +90,31 @@ has explicit **acceptance criteria**. Work top-to-bottom; don't skip M0/M1.
|
|||||||
|
|
||||||
## M4 — Media
|
## M4 — Media
|
||||||
**Goal:** attach + receive + play media.
|
**Goal:** attach + receive + play media.
|
||||||
- [ ] Inbound: `media.upload` chunked → `cache_*_from_bytes` → `media_urls`;
|
- [X] Inbound: `media.upload` chunked → `cache_*_from_bytes` → `media_urls`;
|
||||||
size limit + sha256 + MIME re-sniff.
|
size limit + sha256 + MIME re-sniff.
|
||||||
- [ ] Outbound: `send_*` → `media.offer`; `media.pull` chunked; delivery-path
|
- [X] Outbound: `send_*` → `media.offer`; `media.pull` chunked; delivery-path
|
||||||
security.
|
security.
|
||||||
- [ ] App: SAF pickers + preview chips + upload; `media.pull` → cache;
|
- [X] App: SAF pickers + preview chips + upload; `media.pull` → cache;
|
||||||
**ExoPlayer** inline (audio mini-player, video fullscreen/PiP); image/doc
|
**ExoPlayer** inline (audio mini-player, video fullscreen/PiP); image/doc
|
||||||
viewers.
|
viewers.
|
||||||
- **Demo (on-device):** attach a photo + video (agent sees them); ask agent to
|
- **Demo (on-device):** attach a photo + video (agent sees them); ask agent to
|
||||||
send an image/video → plays live in-app.
|
send an image/video → plays live in-app.
|
||||||
- **Accept:** both directions work; over-limit rejected; playback is live;
|
- **Accept:** both directions work; over-limit rejected; playback is live;
|
||||||
only allowed files are servable.
|
only allowed files are servable.
|
||||||
|
- **Status (complete):** Both directions verified end-to-end on the MIX 2S.
|
||||||
|
Inbound: SAF-picked photo → chunked binary `media.upload` (256 KiB) →
|
||||||
|
gateway size+sha256 verify + MIME re-sniff → `cache_image_from_bytes` →
|
||||||
|
`media.upload.ack` → `message.send` with `media_refs` → agent vision
|
||||||
|
described the image accurately. Outbound: agent `send_image` → `media.offer`
|
||||||
|
→ app auto-`media.pull` (chunked) → cache → image rendered inline. Live
|
||||||
|
playback verified on-device: agent offered a 2s MP4 + 2s MP3 → app pulled
|
||||||
|
both → ExoPlayer video player (blue frame, 00:02/00:02, controls) + audio
|
||||||
|
mini-player rendered and playable. Over-limit rejection, sha256 mismatch,
|
||||||
|
and delivery-path security are covered by the 17-test
|
||||||
|
`tests/gateway/test_android.py` suite (all pass). Note: the app serializes
|
||||||
|
pulls (single `binarySession` slot) via a `Mutex` so concurrent offers don't
|
||||||
|
interleave their byte streams. `media.upload.ack` documented in
|
||||||
|
`04-wire-protocol.md` + `frames.schema.json`.
|
||||||
|
|
||||||
## M5 — Push + offline (FCM + ntfy)
|
## M5 — Push + offline (FCM + ntfy)
|
||||||
**Goal:** reach the phone when backgrounded; catch up on reconnect.
|
**Goal:** reach the phone when backgrounded; catch up on reconnect.
|
||||||
|
|||||||
@@ -68,7 +68,8 @@
|
|||||||
"error": { "payload": { "code": { "type": "string", "enum": ["auth", "not_found", "rate_limited", "media_too_large", "unsupported", "internal"] }, "message": { "type": "string" } } },
|
"error": { "payload": { "code": { "type": "string", "enum": ["auth", "not_found", "rate_limited", "media_too_large", "unsupported", "internal"] }, "message": { "type": "string" } } },
|
||||||
"pong": { "payload": { "ts": { "type": "integer" } } },
|
"pong": { "payload": { "ts": { "type": "integer" } } },
|
||||||
"sync.done": { "payload": { "cursor": { "type": "integer" } } },
|
"sync.done": { "payload": { "cursor": { "type": "integer" } } },
|
||||||
"media.pull.end": { "payload": { "ok": { "type": "boolean" } } }
|
"media.pull.end": { "payload": { "ok": { "type": "boolean" } } },
|
||||||
|
"media.upload.ack": { "description": "Response to media.upload.end; ref is cached and usable in message.send media_refs.", "payload": { "ok": { "type": "boolean" }, "media_ref": { "type": "string" } } }
|
||||||
},
|
},
|
||||||
"app_to_server": {
|
"app_to_server": {
|
||||||
"hello": { "description": "First frame; auth + caps.", "payload": { "token": { "type": "string" }, "device_id": { "type": "string" }, "device_name": { "type": "string" }, "caps": { "type": "object", "properties": { "min_protocol": {"type":"integer"}, "media": {"type":"boolean"}, "push": {"type":"string"} } }, "fcm_token": { "type": "string" }, "ntfy_topic": { "type": "string" } } },
|
"hello": { "description": "First frame; auth + caps.", "payload": { "token": { "type": "string" }, "device_id": { "type": "string" }, "device_name": { "type": "string" }, "caps": { "type": "object", "properties": { "min_protocol": {"type":"integer"}, "media": {"type":"boolean"}, "push": {"type":"string"} } }, "fcm_token": { "type": "string" }, "ntfy_topic": { "type": "string" } } },
|
||||||
|
|||||||
+318
-9
@@ -18,8 +18,17 @@ Milestone M2: agent transparency. ``send()``/``edit_message()`` are mapped to
|
|||||||
``message.start``/``message.update``/``message.stop`` (streaming), tool
|
``message.start``/``message.update``/``message.stop`` (streaming), tool
|
||||||
progress is classified into structured ``tool.start``/``tool.end`` frames,
|
progress is classified into structured ``tool.start``/``tool.end`` frames,
|
||||||
interim commentary becomes ``commentary`` frames, and the code-style
|
interim commentary becomes ``commentary`` frames, and the code-style
|
||||||
reasoning prefix is split into a ``reasoning`` field. Media, outbox, push,
|
reasoning prefix is split into a ``reasoning`` field. Outbox and search land
|
||||||
and search land in later milestones (see ``docs/14-milestones.md``).
|
in M3; media, push, and desktop land in later milestones (see
|
||||||
|
``docs/14-milestones.md``).
|
||||||
|
|
||||||
|
Milestone M4: media. Inbound ``media.upload`` (chunked binary frames) is
|
||||||
|
reassembled in a temp file, verified (size + sha256), re-sniffed, and cached
|
||||||
|
via hermes ``cache_*_from_bytes``; the resulting refs attach to the next
|
||||||
|
``message.send`` as ``MessageEvent.media_urls``. Outbound ``send_*`` calls
|
||||||
|
register the (delivery-validated) file in the media registry and emit
|
||||||
|
``media.offer``; ``media.pull`` streams the file back as chunked binary
|
||||||
|
frames, re-checking ``validate_media_delivery_path`` at pull time.
|
||||||
|
|
||||||
Configuration in config.yaml::
|
Configuration in config.yaml::
|
||||||
|
|
||||||
@@ -87,10 +96,12 @@ from gateway.platforms.base import ( # noqa: E402
|
|||||||
SendResult,
|
SendResult,
|
||||||
MessageEvent,
|
MessageEvent,
|
||||||
MessageType,
|
MessageType,
|
||||||
|
validate_media_delivery_path,
|
||||||
)
|
)
|
||||||
from gateway.config import Platform # noqa: E402
|
from gateway.config import Platform # noqa: E402
|
||||||
from hermes_constants import get_hermes_home # noqa: E402
|
from hermes_constants import get_hermes_home # noqa: E402
|
||||||
|
|
||||||
|
from . import media as media_bridge # noqa: E402
|
||||||
from . import protocol # noqa: E402
|
from . import protocol # noqa: E402
|
||||||
from . import search as search_bridge # noqa: E402
|
from . import search as search_bridge # noqa: E402
|
||||||
from .channels import get_directory # noqa: E402
|
from .channels import get_directory # noqa: E402
|
||||||
@@ -678,6 +689,10 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
get_hermes_home() / "android" / "outbox.db",
|
get_hermes_home() / "android" / "outbox.db",
|
||||||
retention_hours=self.outbox_retention_hours,
|
retention_hours=self.outbox_retention_hours,
|
||||||
)
|
)
|
||||||
|
# M4: media registry (inbound upload refs + outbound offers) and the
|
||||||
|
# last finalized assistant message id per chat (offer association).
|
||||||
|
self._media = media_bridge.MediaStore(get_hermes_home())
|
||||||
|
self._last_message_id: Dict[str, str] = {}
|
||||||
|
|
||||||
def _turn_state(self, chat_id: str) -> _TurnState:
|
def _turn_state(self, chat_id: str) -> _TurnState:
|
||||||
st = self._turns.get(chat_id)
|
st = self._turns.get(chat_id)
|
||||||
@@ -841,6 +856,8 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
ts=int(time.time() * 1000),
|
ts=int(time.time() * 1000),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# M4: media offers emitted after this final associate with it.
|
||||||
|
self._last_message_id[chat_id] = message_id
|
||||||
await self._close_open_tool(chat_id, state, thread_id)
|
await self._close_open_tool(chat_id, state, thread_id)
|
||||||
self._reset_tool_state(state)
|
self._reset_tool_state(state)
|
||||||
state.active = False
|
state.active = False
|
||||||
@@ -901,6 +918,8 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
ts=int(time.time() * 1000),
|
ts=int(time.time() * 1000),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# M4: media offers emitted after this final associate with it.
|
||||||
|
self._last_message_id[chat_id] = message_id
|
||||||
await self._close_open_tool(chat_id, state, thread_id)
|
await self._close_open_tool(chat_id, state, thread_id)
|
||||||
self._reset_tool_state(state)
|
self._reset_tool_state(state)
|
||||||
state.active = False
|
state.active = False
|
||||||
@@ -1049,6 +1068,49 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
"""Clear the typing indicator (``typing`` frame, on=false)."""
|
"""Clear the typing indicator (``typing`` frame, on=false)."""
|
||||||
await self._ws_server.broadcast(protocol.typing(chat_id, False))
|
await self._ws_server.broadcast(protocol.typing(chat_id, False))
|
||||||
|
|
||||||
|
# ── M4: outbound media (agent -> app) ─────────────────────────────────
|
||||||
|
#
|
||||||
|
# The gateway's dispatch partition (gateway/run.py) extracts MEDIA: tags /
|
||||||
|
# image URLs from the final response, filters them through
|
||||||
|
# ``filter_media_delivery_paths``, then calls these ``send_*`` overrides
|
||||||
|
# with local file paths. We re-validate each path (defense in depth),
|
||||||
|
# register it in the media registry, mint a ``media_id``, and emit
|
||||||
|
# ``media.offer``; the app fetches the bytes via ``media.pull``.
|
||||||
|
|
||||||
|
async def _offer_media(
|
||||||
|
self,
|
||||||
|
chat_id: str,
|
||||||
|
path: str,
|
||||||
|
kind: str,
|
||||||
|
filename: Optional[str],
|
||||||
|
metadata: Optional[Dict[str, Any]],
|
||||||
|
) -> SendResult:
|
||||||
|
safe = validate_media_delivery_path(path)
|
||||||
|
if safe is None:
|
||||||
|
logger.warning("android: media path failed delivery validation: %s", path)
|
||||||
|
return SendResult(success=False, error="android: media path not deliverable")
|
||||||
|
try:
|
||||||
|
size = os.path.getsize(safe)
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning("android: media file unreadable %s: %s", safe, e)
|
||||||
|
return SendResult(success=False, error="android: media file unreadable")
|
||||||
|
entry = self._media.register_outbound(
|
||||||
|
safe, kind, media_bridge.mime_for_path(safe), filename or os.path.basename(safe), size
|
||||||
|
)
|
||||||
|
thread_id = _thread_id_from_metadata(metadata)
|
||||||
|
frame = protocol.media_offer(
|
||||||
|
entry.media_id,
|
||||||
|
entry.kind,
|
||||||
|
entry.mime,
|
||||||
|
entry.size,
|
||||||
|
entry.filename,
|
||||||
|
chat_id=chat_id,
|
||||||
|
thread_id=thread_id,
|
||||||
|
message_id=self._last_message_id.get(chat_id),
|
||||||
|
)
|
||||||
|
await self._broadcast_or_log(chat_id, frame)
|
||||||
|
return SendResult(success=True, message_id=entry.media_id)
|
||||||
|
|
||||||
async def send_image(
|
async def send_image(
|
||||||
self,
|
self,
|
||||||
chat_id: str,
|
chat_id: str,
|
||||||
@@ -1057,8 +1119,63 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
reply_to: Optional[str] = None,
|
reply_to: Optional[str] = None,
|
||||||
metadata: Optional[Dict[str, Any]] = None,
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
) -> SendResult:
|
) -> SendResult:
|
||||||
"""Send an image. M1: not implemented (M4)."""
|
"""Send an image (M4: local files offered over WS; remote URLs fall
|
||||||
return SendResult(success=False, error="android: media not implemented yet (M4)")
|
back to the base text rendering)."""
|
||||||
|
if image_url.startswith("file://"):
|
||||||
|
from urllib.parse import unquote
|
||||||
|
return await self._offer_media(chat_id, unquote(image_url[7:]), "image", None, metadata)
|
||||||
|
return await super().send_image(
|
||||||
|
chat_id, image_url, caption=caption, reply_to=reply_to, metadata=metadata
|
||||||
|
)
|
||||||
|
|
||||||
|
async def send_image_file(
|
||||||
|
self,
|
||||||
|
chat_id: str,
|
||||||
|
image_path: str,
|
||||||
|
caption: Optional[str] = None,
|
||||||
|
reply_to: Optional[str] = None,
|
||||||
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> SendResult:
|
||||||
|
"""Send a local image file (M4)."""
|
||||||
|
return await self._offer_media(chat_id, image_path, "image", None, metadata)
|
||||||
|
|
||||||
|
async def send_video(
|
||||||
|
self,
|
||||||
|
chat_id: str,
|
||||||
|
video_path: str,
|
||||||
|
caption: Optional[str] = None,
|
||||||
|
reply_to: Optional[str] = None,
|
||||||
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> SendResult:
|
||||||
|
"""Send a video (M4)."""
|
||||||
|
return await self._offer_media(chat_id, video_path, "video", None, metadata)
|
||||||
|
|
||||||
|
async def send_voice(
|
||||||
|
self,
|
||||||
|
chat_id: str,
|
||||||
|
audio_path: str,
|
||||||
|
caption: Optional[str] = None,
|
||||||
|
reply_to: Optional[str] = None,
|
||||||
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> SendResult:
|
||||||
|
"""Send a voice note / audio file (M4)."""
|
||||||
|
return await self._offer_media(chat_id, audio_path, "voice", None, metadata)
|
||||||
|
|
||||||
|
async def send_document(
|
||||||
|
self,
|
||||||
|
chat_id: str,
|
||||||
|
file_path: str,
|
||||||
|
caption: Optional[str] = None,
|
||||||
|
file_name: Optional[str] = None,
|
||||||
|
reply_to: Optional[str] = None,
|
||||||
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> SendResult:
|
||||||
|
"""Send a document (M4)."""
|
||||||
|
return await self._offer_media(chat_id, file_path, "document", file_name, metadata)
|
||||||
|
|
||||||
# ── Inbound (app -> agent) ────────────────────────────────────────────
|
# ── Inbound (app -> agent) ────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -1068,10 +1185,24 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
Echoes the user message to all devices (multi-device sync + ack),
|
Echoes the user message to all devices (multi-device sync + ack),
|
||||||
then builds a ``MessageEvent`` and hands it to ``handle_message()``
|
then builds a ``MessageEvent`` and hands it to ``handle_message()``
|
||||||
(the gateway's command pipeline + agent turn).
|
(the gateway's command pipeline + agent turn).
|
||||||
|
|
||||||
|
M4: ``media_refs`` reference completed ``media.upload``s; they are
|
||||||
|
resolved to ``MessageEvent.media_urls``/``media_types`` (local paths
|
||||||
|
the agent's vision/audio tools can read) and echoed in the user
|
||||||
|
message's ``media[]`` so every device renders the attachments.
|
||||||
"""
|
"""
|
||||||
payload = frame.payload
|
payload = frame.payload
|
||||||
text = payload.get("text")
|
text = payload.get("text")
|
||||||
if not isinstance(text, str) or not text.strip():
|
text = text if isinstance(text, str) else ""
|
||||||
|
|
||||||
|
refs_raw = payload.get("media_refs")
|
||||||
|
media_refs = (
|
||||||
|
[r for r in refs_raw if isinstance(r, str) and r]
|
||||||
|
if isinstance(refs_raw, list)
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
|
||||||
|
if not text.strip() and not media_refs:
|
||||||
await self._ws_server.send_to(
|
await self._ws_server.send_to(
|
||||||
device_id,
|
device_id,
|
||||||
protocol.error(protocol.ERR_UNSUPPORTED, "message.send requires non-empty text", id=frame.id),
|
protocol.error(protocol.ERR_UNSUPPORTED, "message.send requires non-empty text", id=frame.id),
|
||||||
@@ -1091,6 +1222,30 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
if not isinstance(reply_to, str) or not reply_to.strip():
|
if not isinstance(reply_to, str) or not reply_to.strip():
|
||||||
reply_to = None
|
reply_to = None
|
||||||
|
|
||||||
|
# M4: resolve media refs (single-use; unknown ref -> error).
|
||||||
|
media_urls: List[str] = []
|
||||||
|
media_types: List[str] = []
|
||||||
|
media_wire: List[Dict[str, Any]] = []
|
||||||
|
for ref in media_refs:
|
||||||
|
entry = self._media.get_inbound(ref)
|
||||||
|
if entry is None:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_UNSUPPORTED, f"unknown media_ref {ref}", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
media_urls.append(entry.path)
|
||||||
|
media_types.append(entry.mime)
|
||||||
|
media_wire.append(
|
||||||
|
{
|
||||||
|
"media_id": entry.media_id,
|
||||||
|
"kind": entry.kind,
|
||||||
|
"mime": entry.mime,
|
||||||
|
"size": entry.size,
|
||||||
|
"filename": entry.filename,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
device = self._devices.get(device_id) or {}
|
device = self._devices.get(device_id) or {}
|
||||||
user_name = device.get("name") or device_id
|
user_name = device.get("name") or device_id
|
||||||
|
|
||||||
@@ -1103,10 +1258,31 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
role=protocol.ROLE_USER,
|
role=protocol.ROLE_USER,
|
||||||
text=text,
|
text=text,
|
||||||
thread_id=thread_id,
|
thread_id=thread_id,
|
||||||
|
media=media_wire or None,
|
||||||
reply_to=reply_to,
|
reply_to=reply_to,
|
||||||
ts=int(time.time() * 1000),
|
ts=int(time.time() * 1000),
|
||||||
)
|
)
|
||||||
await self._ws_server.broadcast(echo)
|
await self._ws_server.broadcast(echo)
|
||||||
|
# Refs are consumed by this message (no replay).
|
||||||
|
for ref in media_refs:
|
||||||
|
self._media.pop_inbound(ref)
|
||||||
|
|
||||||
|
# M4: a new user turn starts -- stale offer association is dropped.
|
||||||
|
self._last_message_id.pop(chat_id, None)
|
||||||
|
|
||||||
|
kind = media_wire[0]["kind"] if media_wire else None
|
||||||
|
if kind == "image":
|
||||||
|
message_type = MessageType.PHOTO
|
||||||
|
elif kind == "video":
|
||||||
|
message_type = MessageType.VIDEO
|
||||||
|
elif kind == "audio":
|
||||||
|
message_type = MessageType.AUDIO
|
||||||
|
elif kind == "voice":
|
||||||
|
message_type = MessageType.VOICE
|
||||||
|
elif kind == "document":
|
||||||
|
message_type = MessageType.DOCUMENT
|
||||||
|
else:
|
||||||
|
message_type = MessageType.TEXT
|
||||||
|
|
||||||
source = self.build_source(
|
source = self.build_source(
|
||||||
chat_id=chat_id,
|
chat_id=chat_id,
|
||||||
@@ -1118,15 +1294,143 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
)
|
)
|
||||||
event = MessageEvent(
|
event = MessageEvent(
|
||||||
text=text,
|
text=text,
|
||||||
message_type=MessageType.TEXT,
|
message_type=message_type,
|
||||||
user_id=device_id,
|
user_id=device_id,
|
||||||
user_name=user_name,
|
user_name=user_name,
|
||||||
source=source,
|
source=source,
|
||||||
message_id=message_id,
|
message_id=message_id,
|
||||||
reply_to_message_id=reply_to,
|
reply_to_message_id=reply_to,
|
||||||
|
media_urls=media_urls,
|
||||||
|
media_types=media_types,
|
||||||
)
|
)
|
||||||
await self.handle_message(event)
|
await self.handle_message(event)
|
||||||
|
|
||||||
|
# ── M4: inbound media (app -> agent) ──────────────────────────────────
|
||||||
|
#
|
||||||
|
# ``media.upload.start`` -> raw binary frames (one at a time per
|
||||||
|
# connection) -> ``media.upload.end``. The session streams to a temp
|
||||||
|
# file (bounded RAM); on end we verify size + sha256, re-sniff the kind,
|
||||||
|
# and cache via hermes ``cache_*_from_bytes``. ``media.pull`` serves an
|
||||||
|
# outbound offer as chunked binary frames, re-checking the delivery-path
|
||||||
|
# validation at pull time.
|
||||||
|
|
||||||
|
async def on_media_upload_start(self, frame: protocol.Frame, device_id: str) -> None:
|
||||||
|
payload = frame.payload
|
||||||
|
media_ref = str(payload.get("media_ref") or "").strip()
|
||||||
|
if not media_ref or len(media_ref) > 64:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.start requires media_ref", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
kind = payload.get("kind")
|
||||||
|
if kind not in media_bridge.KINDS:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_UNSUPPORTED, f"unsupported media kind {kind!r}", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
mime = str(payload.get("mime") or "application/octet-stream")[:128]
|
||||||
|
filename = str(payload.get("filename") or "upload")[:255]
|
||||||
|
size = payload.get("size")
|
||||||
|
try:
|
||||||
|
size = int(size) if size is not None else -1
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
size = -1
|
||||||
|
if size <= 0:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.start requires a positive size", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if size > self.max_upload_bytes:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(
|
||||||
|
protocol.ERR_MEDIA_TOO_LARGE,
|
||||||
|
f"upload of {size} bytes exceeds limit ({self.max_upload_bytes})",
|
||||||
|
id=frame.id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._media.create_upload(
|
||||||
|
device_id, media_ref, kind, mime, filename, size, frame.id,
|
||||||
|
self.max_upload_bytes,
|
||||||
|
)
|
||||||
|
except media_bridge.MediaError as e:
|
||||||
|
await self._ws_server.send_to(device_id, protocol.error(e.code, e.message, id=frame.id))
|
||||||
|
return
|
||||||
|
# No ack: WS ordering guarantees the server processes this before the
|
||||||
|
# first binary chunk; failures arrive as ``error`` frames.
|
||||||
|
|
||||||
|
async def on_media_chunk(self, device_id: str, chunk: bytes) -> None:
|
||||||
|
session = self._media.get_upload(device_id)
|
||||||
|
if session is None:
|
||||||
|
return # stray binary frame: ignore (forward-compat)
|
||||||
|
session.feed(chunk)
|
||||||
|
if session.failed:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(session.error_code, session.error_message, id=session.request_id),
|
||||||
|
)
|
||||||
|
self._media.discard_upload(device_id, session.media_ref)
|
||||||
|
|
||||||
|
async def on_media_upload_end(self, frame: protocol.Frame, device_id: str) -> None:
|
||||||
|
payload = frame.payload
|
||||||
|
media_ref = str(payload.get("media_ref") or "").strip()
|
||||||
|
sha256 = str(payload.get("sha256") or "").strip().lower()
|
||||||
|
if not media_ref:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_UNSUPPORTED, "media.upload.end requires media_ref", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
entry = self._media.complete_upload(device_id, media_ref, sha256)
|
||||||
|
except media_bridge.MediaError as e:
|
||||||
|
await self._ws_server.send_to(device_id, protocol.error(e.code, e.message, id=frame.id))
|
||||||
|
return
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id, protocol.media_upload_ack(True, entry.media_id, id=frame.id)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def on_media_pull(self, frame: protocol.Frame, device_id: str) -> None:
|
||||||
|
payload = frame.payload
|
||||||
|
media_id = str(payload.get("media_id") or "").strip()
|
||||||
|
entry = self._media.get_outbound(media_id) if media_id else None
|
||||||
|
if entry is None:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_NOT_FOUND, f"unknown media_id {media_id!r}", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
# Delivery-path security: re-validate at pull time (the file may have
|
||||||
|
# moved / been replaced since the offer).
|
||||||
|
safe = validate_media_delivery_path(entry.path)
|
||||||
|
if safe is None:
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id,
|
||||||
|
protocol.error(protocol.ERR_NOT_FOUND, "media no longer deliverable", id=frame.id),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
conn = self._ws_server.connection(device_id)
|
||||||
|
if conn is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await media_bridge.stream_file(conn.ws, safe, media_bridge.DEFAULT_CHUNK_BYTES)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("android: media.pull stream failed for %s: %s", media_id, e)
|
||||||
|
await self._ws_server.send_to(
|
||||||
|
device_id, protocol.error(protocol.ERR_INTERNAL, f"pull failed: {e}", id=frame.id)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
await self._ws_server.send_to(device_id, protocol.media_pull_end(True, id=frame.id))
|
||||||
|
|
||||||
|
def on_connection_closed(self, device_id: str) -> None:
|
||||||
|
"""M4: drop in-flight upload temp files for a disconnected device."""
|
||||||
|
self._media.discard_device(device_id)
|
||||||
|
|
||||||
# ── M3: channel directory management (app -> agent) ───────────────────
|
# ── M3: channel directory management (app -> agent) ───────────────────
|
||||||
#
|
#
|
||||||
# Each request is answered by broadcasting the matching ``channel.*``
|
# Each request is answered by broadcasting the matching ``channel.*``
|
||||||
@@ -1318,12 +1622,12 @@ class AndroidAdapter(BasePlatformAdapter):
|
|||||||
# ── hello.ack helpers ─────────────────────────────────────────────────
|
# ── hello.ack helpers ─────────────────────────────────────────────────
|
||||||
|
|
||||||
def server_caps(self) -> Dict[str, Any]:
|
def server_caps(self) -> Dict[str, Any]:
|
||||||
"""Capability flags advertised in ``hello.ack`` (M3 surface)."""
|
"""Capability flags advertised in ``hello.ack`` (M4 surface)."""
|
||||||
return {
|
return {
|
||||||
"streaming": True, # M2: message.start/update/stop
|
"streaming": True, # M2: message.start/update/stop
|
||||||
"reasoning": True, # M2: reasoning field on message / message.stop
|
"reasoning": True, # M2: reasoning field on message / message.stop
|
||||||
"tools": True, # M2: tool.start/progress/end
|
"tools": True, # M2: tool.start/progress/end
|
||||||
"media": False, # M4
|
"media": True, # M4: media.upload/offer/pull
|
||||||
"search": True, # M3: search frame
|
"search": True, # M3: search frame
|
||||||
"push": self.push_backend,
|
"push": self.push_backend,
|
||||||
"pickers": False, # M2+
|
"pickers": False, # M2+
|
||||||
@@ -1431,6 +1735,11 @@ def register(ctx):
|
|||||||
"(Android/Desktop). It renders Markdown, inline code, images, "
|
"(Android/Desktop). It renders Markdown, inline code, images, "
|
||||||
"audio and video, and shows your reasoning and tool activity. "
|
"audio and video, and shows your reasoning and tool activity. "
|
||||||
"Conversations are organized into channels and optional threads. "
|
"Conversations are organized into channels and optional threads. "
|
||||||
"Keep formatting rich but readable."
|
"Keep formatting rich but readable. "
|
||||||
|
"You can send media files natively: to deliver a file to the user, "
|
||||||
|
"include MEDIA:/absolute/path/to/file in your response. Images "
|
||||||
|
"(.png, .jpg, .webp) appear as photos, audio (.ogg, .mp3, .m4a) "
|
||||||
|
"plays inline, videos (.mp4, .webm, .mov) play inline, and other "
|
||||||
|
"files arrive as downloadable documents."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
+440
-8
@@ -2,15 +2,447 @@
|
|||||||
|
|
||||||
Inbound: ``media.upload`` (chunked binary frames) -> ``cache_*_from_bytes``
|
Inbound: ``media.upload`` (chunked binary frames) -> ``cache_*_from_bytes``
|
||||||
-> a ``media_ref`` the adapter attaches to the ``MessageEvent``. Enforces
|
-> a ``media_ref`` the adapter attaches to the ``MessageEvent``. Enforces
|
||||||
size limit + sha256 + MIME re-sniff.
|
size limit + sha256 + MIME re-sniff (the client's declared kind is not
|
||||||
|
trusted). Chunks are streamed to a temp file so RAM stays bounded during
|
||||||
|
the transfer; the file is read once at the end for the hermes cache
|
||||||
|
helpers.
|
||||||
|
|
||||||
Outbound: ``send_*`` -> stage the file in the media cache, mint a
|
Outbound: ``send_*`` -> register the (already delivery-validated) file in
|
||||||
``media_id``, emit ``media.offer {media_id, mime, size, filename, kind}``;
|
the media registry, mint a ``media_id``, emit ``media.offer {media_id,
|
||||||
serve bytes on ``media.pull`` as chunked binary frames. Delivery-path
|
kind, mime, size, filename}``; serve bytes on ``media.pull`` as chunked
|
||||||
security via ``validate_media_delivery_path``.
|
binary frames. Delivery-path security via ``validate_media_delivery_path``
|
||||||
|
(re-checked at pull time, so only allowed files are servable).
|
||||||
|
|
||||||
Reuses hermes ``cache_image/audio/video/document_from_bytes`` where possible.
|
Reuses hermes ``cache_image/audio/video/document_from_bytes`` + the
|
||||||
All paths under ``get_hermes_home()/"android"/media``.
|
``_looks_like_image`` / ``sniff_container`` magic-byte sniffers. Temp files
|
||||||
|
live under ``get_hermes_home()/"android"/media/tmp``.
|
||||||
|
|
||||||
Milestone M4.
|
Milestone M4.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Optional, Tuple
|
||||||
|
|
||||||
|
from gateway.platforms.base import (
|
||||||
|
_looks_like_image,
|
||||||
|
cache_audio_from_bytes,
|
||||||
|
cache_document_from_bytes,
|
||||||
|
cache_image_from_bytes,
|
||||||
|
cache_video_from_bytes,
|
||||||
|
)
|
||||||
|
from tools.audio_container import sniff_container
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# One WS binary frame carries at most this many media bytes (docs/07 §7.5).
|
||||||
|
DEFAULT_CHUNK_BYTES = 256 * 1024
|
||||||
|
|
||||||
|
KINDS = ("image", "audio", "video", "document", "voice")
|
||||||
|
|
||||||
|
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
|
||||||
|
# Magic-byte containers that are unambiguously audio (vs video-in-same-box).
|
||||||
|
_AUDIO_CONTAINERS = {"m4a", "ogg", "flac", "wav", "mp3", "aac"}
|
||||||
|
_VIDEO_CONTAINERS = {"mp4", "webm"}
|
||||||
|
|
||||||
|
# Extension -> MIME for outbound offers (the app picks a player/viewer from it).
|
||||||
|
_EXT_TO_MIME: Dict[str, str] = {
|
||||||
|
".jpg": "image/jpeg",
|
||||||
|
".jpeg": "image/jpeg",
|
||||||
|
".png": "image/png",
|
||||||
|
".webp": "image/webp",
|
||||||
|
".gif": "image/gif",
|
||||||
|
".bmp": "image/bmp",
|
||||||
|
".heic": "image/heic",
|
||||||
|
".mp4": "video/mp4",
|
||||||
|
".mov": "video/quicktime",
|
||||||
|
".webm": "video/webm",
|
||||||
|
".mkv": "video/x-matroska",
|
||||||
|
".avi": "video/x-msvideo",
|
||||||
|
".3gp": "video/3gpp",
|
||||||
|
".ogg": "audio/ogg",
|
||||||
|
".opus": "audio/ogg",
|
||||||
|
".m4a": "audio/mp4",
|
||||||
|
".mp3": "audio/mpeg",
|
||||||
|
".wav": "audio/wav",
|
||||||
|
".flac": "audio/flac",
|
||||||
|
".aac": "audio/aac",
|
||||||
|
".pdf": "application/pdf",
|
||||||
|
".txt": "text/plain",
|
||||||
|
".md": "text/markdown",
|
||||||
|
".csv": "text/csv",
|
||||||
|
".html": "text/html",
|
||||||
|
".json": "application/json",
|
||||||
|
".zip": "application/zip",
|
||||||
|
".doc": "application/msword",
|
||||||
|
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
".xls": "application/vnd.ms-excel",
|
||||||
|
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
".ppt": "application/vnd.ms-powerpoint",
|
||||||
|
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||||
|
}
|
||||||
|
|
||||||
|
# MIME -> extension for inbound caching (the cache helpers take an ext).
|
||||||
|
_MIME_TO_EXT: Dict[str, str] = {
|
||||||
|
"image/jpeg": ".jpg",
|
||||||
|
"image/png": ".png",
|
||||||
|
"image/webp": ".webp",
|
||||||
|
"image/gif": ".gif",
|
||||||
|
"image/bmp": ".bmp",
|
||||||
|
"image/heic": ".heic",
|
||||||
|
"video/mp4": ".mp4",
|
||||||
|
"video/quicktime": ".mov",
|
||||||
|
"video/webm": ".webm",
|
||||||
|
"video/x-matroska": ".mkv",
|
||||||
|
"video/x-msvideo": ".avi",
|
||||||
|
"audio/ogg": ".ogg",
|
||||||
|
"audio/mp4": ".m4a",
|
||||||
|
"audio/mpeg": ".mp3",
|
||||||
|
"audio/wav": ".wav",
|
||||||
|
"audio/flac": ".flac",
|
||||||
|
"audio/aac": ".aac",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MediaError(Exception):
|
||||||
|
"""Media flow failure carrying a wire ``error`` code."""
|
||||||
|
|
||||||
|
def __init__(self, code: str, message: str):
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
def mime_for_path(path: str) -> str:
|
||||||
|
ext = os.path.splitext(path)[1].lower()
|
||||||
|
return _EXT_TO_MIME.get(ext, "application/octet-stream")
|
||||||
|
|
||||||
|
|
||||||
|
def ext_for_mime(mime: str, filename: str, default: str) -> str:
|
||||||
|
"""Best-effort extension for the hermes cache helpers.
|
||||||
|
|
||||||
|
Prefers the MIME map, falls back to the filename's own extension, then
|
||||||
|
*default*.
|
||||||
|
"""
|
||||||
|
base = (mime or "").split(";")[0].strip().lower()
|
||||||
|
ext = _MIME_TO_EXT.get(base)
|
||||||
|
if ext:
|
||||||
|
return ext
|
||||||
|
file_ext = os.path.splitext(filename or "")[1].lower()
|
||||||
|
if file_ext and len(file_ext) <= 6:
|
||||||
|
return file_ext
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def reclassify_kind(head: bytes, declared_kind: str) -> str:
|
||||||
|
"""Re-sniff the real kind from magic bytes (don't trust the client).
|
||||||
|
|
||||||
|
Images are detected first (RIFF/WEBP would otherwise look like a
|
||||||
|
container); then AV containers. Video declared by the client is kept
|
||||||
|
when no magic matches (some codecs have no leading magic we check);
|
||||||
|
anything else falls back to ``document``.
|
||||||
|
"""
|
||||||
|
if _looks_like_image(head):
|
||||||
|
return "image"
|
||||||
|
container = sniff_container(head)
|
||||||
|
if container in _AUDIO_CONTAINERS:
|
||||||
|
return "voice" if declared_kind == "voice" else "audio"
|
||||||
|
if container in _VIDEO_CONTAINERS:
|
||||||
|
return "video"
|
||||||
|
return declared_kind if declared_kind == "video" else "document"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MediaEntry:
|
||||||
|
"""One servable/cached media object (inbound ref or outbound offer)."""
|
||||||
|
|
||||||
|
media_id: str # mu_* (inbound ref, app-chosen) or md_* (outbound, minted)
|
||||||
|
kind: str
|
||||||
|
mime: str
|
||||||
|
size: int
|
||||||
|
filename: str
|
||||||
|
path: str # absolute local path
|
||||||
|
ts: float = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
class UploadSession:
|
||||||
|
"""One in-flight ``media.upload`` (per connection; one at a time).
|
||||||
|
|
||||||
|
Chunks are appended to a temp file (bounded RAM); the sha256 is
|
||||||
|
accumulated incrementally. The declared size is enforced as bytes
|
||||||
|
arrive so an over-limit transfer is rejected early.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
media_ref: str,
|
||||||
|
kind: str,
|
||||||
|
mime: str,
|
||||||
|
filename: str,
|
||||||
|
declared_size: int,
|
||||||
|
request_id: Optional[int],
|
||||||
|
max_bytes: int,
|
||||||
|
tmp_dir: Path,
|
||||||
|
):
|
||||||
|
self.media_ref = media_ref
|
||||||
|
self.kind = kind
|
||||||
|
self.mime = mime
|
||||||
|
self.filename = filename
|
||||||
|
self.declared_size = declared_size
|
||||||
|
self.request_id = request_id
|
||||||
|
self.max_bytes = max_bytes
|
||||||
|
self.received = 0
|
||||||
|
self.failed = False
|
||||||
|
self.error_code = ""
|
||||||
|
self.error_message = ""
|
||||||
|
self._sha = hashlib.sha256()
|
||||||
|
fd, self.tmp_path = tempfile.mkstemp(prefix="upl_", dir=str(tmp_dir))
|
||||||
|
self._fh = os.fdopen(fd, "wb")
|
||||||
|
|
||||||
|
def feed(self, chunk: bytes) -> None:
|
||||||
|
if self.failed:
|
||||||
|
return
|
||||||
|
self.received += len(chunk)
|
||||||
|
if self.max_bytes and self.received > self.max_bytes:
|
||||||
|
self._fail(
|
||||||
|
"media_too_large",
|
||||||
|
f"upload exceeds limit ({self.received} > {self.max_bytes} bytes)",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._fh.write(chunk)
|
||||||
|
self._fh.flush()
|
||||||
|
self._sha.update(chunk)
|
||||||
|
except OSError as e:
|
||||||
|
self._fail("internal", f"temp file write failed: {e}")
|
||||||
|
|
||||||
|
def _fail(self, code: str, message: str) -> None:
|
||||||
|
self.failed = True
|
||||||
|
self.error_code = code
|
||||||
|
self.error_message = message
|
||||||
|
logger.warning("android: upload %s failed: %s", self.media_ref, message)
|
||||||
|
|
||||||
|
def digest(self) -> str:
|
||||||
|
return self._sha.hexdigest()
|
||||||
|
|
||||||
|
def read_bytes(self) -> bytes:
|
||||||
|
self._fh.close()
|
||||||
|
with open(self.tmp_path, "rb") as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Discard the session and remove the temp file."""
|
||||||
|
try:
|
||||||
|
self._fh.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
os.unlink(self.tmp_path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class MediaStore:
|
||||||
|
"""In-memory media registry + upload session tracking.
|
||||||
|
|
||||||
|
Inbound refs (``mu_*``, app-chosen) are short-lived: created on
|
||||||
|
``media.upload.end``, consumed by the next ``message.send``. Outbound
|
||||||
|
offers (``md_*``, minted here) live until pulled (or process restart —
|
||||||
|
the outbox replays the offer frame, but the id->path map is in-memory
|
||||||
|
for M4).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, hermes_home: Path):
|
||||||
|
self._tmp_dir = hermes_home / "android" / "media" / "tmp"
|
||||||
|
self._tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
# (device_id, media_ref) -> UploadSession (one active per device)
|
||||||
|
self._uploads: Dict[Tuple[str, str], UploadSession] = {}
|
||||||
|
self._inbound: Dict[str, MediaEntry] = {}
|
||||||
|
self._outbound: Dict[str, MediaEntry] = {}
|
||||||
|
|
||||||
|
# ── Inbound uploads ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def create_upload(
|
||||||
|
self,
|
||||||
|
device_id: str,
|
||||||
|
media_ref: str,
|
||||||
|
kind: str,
|
||||||
|
mime: str,
|
||||||
|
filename: str,
|
||||||
|
declared_size: int,
|
||||||
|
request_id: Optional[int],
|
||||||
|
max_bytes: int,
|
||||||
|
) -> UploadSession:
|
||||||
|
with self._lock:
|
||||||
|
for (dev, _ref), sess in self._uploads.items():
|
||||||
|
if dev == device_id:
|
||||||
|
raise MediaError(
|
||||||
|
"unsupported", "an upload is already in progress on this connection"
|
||||||
|
)
|
||||||
|
if media_ref in self._inbound:
|
||||||
|
raise MediaError("unsupported", f"media_ref {media_ref} already used")
|
||||||
|
sess = UploadSession(
|
||||||
|
media_ref, kind, mime, filename, declared_size, request_id,
|
||||||
|
max_bytes, self._tmp_dir,
|
||||||
|
)
|
||||||
|
self._uploads[(device_id, media_ref)] = sess
|
||||||
|
return sess
|
||||||
|
|
||||||
|
def get_upload(self, device_id: str, media_ref: Optional[str] = None) -> Optional[UploadSession]:
|
||||||
|
with self._lock:
|
||||||
|
if media_ref is not None:
|
||||||
|
return self._uploads.get((device_id, media_ref))
|
||||||
|
for (dev, _ref), sess in self._uploads.items():
|
||||||
|
if dev == device_id:
|
||||||
|
return sess
|
||||||
|
return None
|
||||||
|
|
||||||
|
def discard_upload(self, device_id: str, media_ref: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
sess = self._uploads.pop((device_id, media_ref), None)
|
||||||
|
if sess is not None:
|
||||||
|
sess.close()
|
||||||
|
|
||||||
|
def complete_upload(self, device_id: str, media_ref: str, sha256_hex: str) -> MediaEntry:
|
||||||
|
"""Verify + cache a finished upload. Raises ``MediaError`` on failure."""
|
||||||
|
with self._lock:
|
||||||
|
sess = self._uploads.pop((device_id, media_ref), None)
|
||||||
|
if sess is None:
|
||||||
|
raise MediaError("not_found", f"no active upload for media_ref {media_ref}")
|
||||||
|
try:
|
||||||
|
if sess.failed:
|
||||||
|
raise MediaError(sess.error_code, sess.error_message)
|
||||||
|
if sess.received != sess.declared_size:
|
||||||
|
raise MediaError(
|
||||||
|
"internal",
|
||||||
|
f"size mismatch (declared {sess.declared_size}, received {sess.received})",
|
||||||
|
)
|
||||||
|
if not _SHA256_RE.match(sha256_hex or ""):
|
||||||
|
raise MediaError("unsupported", "sha256 must be 64 hex chars")
|
||||||
|
if sess.digest() != sha256_hex:
|
||||||
|
raise MediaError("internal", "sha256 mismatch")
|
||||||
|
|
||||||
|
data = sess.read_bytes()
|
||||||
|
kind = reclassify_kind(data[:65536], sess.kind)
|
||||||
|
try:
|
||||||
|
if kind == "image":
|
||||||
|
path = cache_image_from_bytes(
|
||||||
|
data, ext_for_mime(sess.mime, sess.filename, ".jpg")
|
||||||
|
)
|
||||||
|
elif kind in ("audio", "voice"):
|
||||||
|
path = cache_audio_from_bytes(
|
||||||
|
data, ext_for_mime(sess.mime, sess.filename, ".ogg")
|
||||||
|
)
|
||||||
|
elif kind == "video":
|
||||||
|
path = cache_video_from_bytes(
|
||||||
|
data, ext_for_mime(sess.mime, sess.filename, ".mp4")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
path = cache_document_from_bytes(data, sess.filename or "document")
|
||||||
|
except ValueError as e:
|
||||||
|
# hermes cap (gateway.max_inbound_media_bytes) or a
|
||||||
|
# non-image payload masquerading as an image.
|
||||||
|
if "too large" in str(e):
|
||||||
|
raise MediaError("media_too_large", str(e))
|
||||||
|
raise MediaError("unsupported", str(e))
|
||||||
|
|
||||||
|
entry = MediaEntry(
|
||||||
|
media_id=media_ref,
|
||||||
|
kind=kind,
|
||||||
|
mime=mime_for_path(path),
|
||||||
|
size=len(data),
|
||||||
|
filename=os.path.basename(path),
|
||||||
|
path=path,
|
||||||
|
ts=time.time(),
|
||||||
|
)
|
||||||
|
with self._lock:
|
||||||
|
self._inbound[media_ref] = entry
|
||||||
|
logger.info(
|
||||||
|
"android: upload %s cached as %s (%s, %d bytes)",
|
||||||
|
media_ref, kind, path, len(data),
|
||||||
|
)
|
||||||
|
return entry
|
||||||
|
finally:
|
||||||
|
sess.close()
|
||||||
|
|
||||||
|
def get_inbound(self, media_ref: str) -> Optional[MediaEntry]:
|
||||||
|
with self._lock:
|
||||||
|
return self._inbound.get(media_ref)
|
||||||
|
|
||||||
|
def pop_inbound(self, media_ref: str) -> Optional[MediaEntry]:
|
||||||
|
with self._lock:
|
||||||
|
return self._inbound.pop(media_ref, None)
|
||||||
|
|
||||||
|
# ── Outbound offers ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def register_outbound(
|
||||||
|
self, path: str, kind: str, mime: str, filename: str, size: int
|
||||||
|
) -> MediaEntry:
|
||||||
|
entry = MediaEntry(
|
||||||
|
media_id=f"md_{uuid.uuid4().hex[:12]}",
|
||||||
|
kind=kind,
|
||||||
|
mime=mime,
|
||||||
|
size=size,
|
||||||
|
filename=filename or os.path.basename(path),
|
||||||
|
path=path,
|
||||||
|
ts=time.time(),
|
||||||
|
)
|
||||||
|
with self._lock:
|
||||||
|
self._outbound[entry.media_id] = entry
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def get_outbound(self, media_id: str) -> Optional[MediaEntry]:
|
||||||
|
with self._lock:
|
||||||
|
return self._outbound.get(media_id)
|
||||||
|
|
||||||
|
# ── Cleanup ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def discard_device(self, device_id: str) -> None:
|
||||||
|
"""Drop all in-flight uploads for a disconnected device."""
|
||||||
|
with self._lock:
|
||||||
|
stale = [k for k in self._uploads if k[0] == device_id]
|
||||||
|
sessions = [self._uploads.pop(k) for k in stale]
|
||||||
|
for sess in sessions:
|
||||||
|
sess.close()
|
||||||
|
|
||||||
|
def prune_outbound(self, max_age_s: float = 24 * 3600) -> int:
|
||||||
|
"""Drop offers older than *max_age_s* (their files stay in the hermes
|
||||||
|
cache; only the id->path map is pruned)."""
|
||||||
|
now = time.time()
|
||||||
|
with self._lock:
|
||||||
|
stale = [k for k, e in self._outbound.items() if now - e.ts > max_age_s]
|
||||||
|
for k in stale:
|
||||||
|
del self._outbound[k]
|
||||||
|
return len(stale)
|
||||||
|
|
||||||
|
|
||||||
|
async def stream_file(
|
||||||
|
ws, path: str, chunk_bytes: int = DEFAULT_CHUNK_BYTES, timeout: float = 10.0
|
||||||
|
) -> int:
|
||||||
|
"""Stream *path* to *ws* as binary frames. Returns bytes sent.
|
||||||
|
|
||||||
|
Ordering is guaranteed by the WebSocket; the caller sends the terminal
|
||||||
|
``media.pull.end`` frame afterwards. Each chunk send is bounded by
|
||||||
|
*timeout* so a stalled puller can't wedge the handler forever (the
|
||||||
|
caller treats the raised error as an aborted pull).
|
||||||
|
"""
|
||||||
|
sent = 0
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
while True:
|
||||||
|
chunk = f.read(chunk_bytes)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
await asyncio.wait_for(ws.send(chunk), timeout=timeout)
|
||||||
|
sent += len(chunk)
|
||||||
|
return sent
|
||||||
@@ -68,6 +68,14 @@ TYPE_SEARCH_RESULTS = "search.results"
|
|||||||
TYPE_SYNC = "sync"
|
TYPE_SYNC = "sync"
|
||||||
TYPE_SYNC_DONE = "sync.done"
|
TYPE_SYNC_DONE = "sync.done"
|
||||||
|
|
||||||
|
# Media (M4)
|
||||||
|
TYPE_MEDIA_UPLOAD_START = "media.upload.start"
|
||||||
|
TYPE_MEDIA_UPLOAD_END = "media.upload.end"
|
||||||
|
TYPE_MEDIA_UPLOAD_ACK = "media.upload.ack"
|
||||||
|
TYPE_MEDIA_OFFER = "media.offer"
|
||||||
|
TYPE_MEDIA_PULL = "media.pull"
|
||||||
|
TYPE_MEDIA_PULL_END = "media.pull.end"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Error codes (``error`` frame payload.code)
|
# Error codes (``error`` frame payload.code)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -444,6 +452,53 @@ def sync_done(cursor: int, *, id: Optional[int] = None) -> Frame:
|
|||||||
return Frame(type=TYPE_SYNC_DONE, id=id, payload={"cursor": cursor})
|
return Frame(type=TYPE_SYNC_DONE, id=id, payload={"cursor": cursor})
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Media frames (M4)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def media_offer(
|
||||||
|
media_id: str,
|
||||||
|
kind: str,
|
||||||
|
mime: str,
|
||||||
|
size: int,
|
||||||
|
filename: str,
|
||||||
|
*,
|
||||||
|
chat_id: Optional[str] = None,
|
||||||
|
thread_id: Optional[str] = None,
|
||||||
|
message_id: Optional[str] = None,
|
||||||
|
) -> Frame:
|
||||||
|
"""Event: the agent produced media the app can fetch via ``media.pull``.
|
||||||
|
|
||||||
|
``message_id`` (optional) associates the offer with the assistant message
|
||||||
|
it belongs to (the app falls back to the lane's last assistant message).
|
||||||
|
"""
|
||||||
|
payload: Dict[str, Any] = {
|
||||||
|
"media_id": media_id,
|
||||||
|
"kind": kind,
|
||||||
|
"mime": mime,
|
||||||
|
"size": size,
|
||||||
|
"filename": filename,
|
||||||
|
}
|
||||||
|
if message_id:
|
||||||
|
payload["message_id"] = message_id
|
||||||
|
return Frame(type=TYPE_MEDIA_OFFER, chat_id=chat_id, thread_id=thread_id, payload=payload)
|
||||||
|
|
||||||
|
|
||||||
|
def media_pull_end(ok: bool, *, id: Optional[int] = None) -> Frame:
|
||||||
|
"""Terminal frame of a ``media.pull`` binary stream."""
|
||||||
|
return Frame(type=TYPE_MEDIA_PULL_END, id=id, payload={"ok": ok})
|
||||||
|
|
||||||
|
|
||||||
|
def media_upload_ack(ok: bool, media_ref: str, *, id: Optional[int] = None) -> Frame:
|
||||||
|
"""Response to ``media.upload.end``: the ref is cached and may be used in
|
||||||
|
a ``message.send`` ``media_refs``. Failures use ``error`` frames instead."""
|
||||||
|
return Frame(
|
||||||
|
type=TYPE_MEDIA_UPLOAD_ACK,
|
||||||
|
id=id,
|
||||||
|
payload={"ok": ok, "media_ref": media_ref},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def error(code: str, message: str, *, id: Optional[int] = None) -> Frame:
|
def error(code: str, message: str, *, id: Optional[int] = None) -> Frame:
|
||||||
return Frame(type=TYPE_ERROR, id=id, payload={"code": code, "message": message})
|
return Frame(type=TYPE_ERROR, id=id, payload={"code": code, "message": message})
|
||||||
|
|
||||||
|
|||||||
@@ -16,13 +16,19 @@ Options:
|
|||||||
--token ANDROID_TOKEN (default: $ANDROID_TOKEN)
|
--token ANDROID_TOKEN (default: $ANDROID_TOKEN)
|
||||||
--device device_id (default: probe-<rand>)
|
--device device_id (default: probe-<rand>)
|
||||||
--send TEXT send this message after pairing (default: "hello")
|
--send TEXT send this message after pairing (default: "hello")
|
||||||
|
--upload F M4: upload F (chunked media.upload) and attach it to the
|
||||||
|
message.send via media_refs
|
||||||
|
--pull-offer M4: when a media.offer arrives during the turn, pull the
|
||||||
|
media (chunked) and verify the byte count
|
||||||
--timeout S seconds to wait for the final reply (default 120)
|
--timeout S seconds to wait for the final reply (default 120)
|
||||||
--authfail expect an auth rejection (wrong token) and exit 0 on it
|
--authfail expect an auth rejection (wrong token) and exit 0 on it
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -80,12 +86,98 @@ def _print_frame(raw):
|
|||||||
extra = f" on={payload.get('on')}"
|
extra = f" on={payload.get('on')}"
|
||||||
elif ftype == "pong":
|
elif ftype == "pong":
|
||||||
extra = ""
|
extra = ""
|
||||||
|
elif ftype == "media.offer":
|
||||||
|
extra = (f" media_id={payload.get('media_id')} kind={payload.get('kind')} "
|
||||||
|
f"mime={payload.get('mime')} size={payload.get('size')} "
|
||||||
|
f"file={payload.get('filename')!r} msg={payload.get('message_id')}")
|
||||||
|
elif ftype == "media.upload.ack":
|
||||||
|
extra = f" ok={payload.get('ok')} ref={payload.get('media_ref')}"
|
||||||
|
elif ftype == "media.pull.end":
|
||||||
|
extra = f" ok={payload.get('ok')}"
|
||||||
scope = f" chat={chat}" if chat else ""
|
scope = f" chat={chat}" if chat else ""
|
||||||
idpart = f" id={fid}" if fid is not None else ""
|
idpart = f" id={fid}" if fid is not None else ""
|
||||||
print(f" <- {ftype}{idpart}{scope}{extra}")
|
print(f" <- {ftype}{idpart}{scope}{extra}")
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _kind_for_path(path: str) -> str:
|
||||||
|
mime, _ = mimetypes.guess_type(path)
|
||||||
|
mime = mime or "application/octet-stream"
|
||||||
|
if mime.startswith("image/"):
|
||||||
|
return "image"
|
||||||
|
if mime.startswith("video/"):
|
||||||
|
return "video"
|
||||||
|
if mime.startswith("audio/"):
|
||||||
|
return "audio"
|
||||||
|
return "document"
|
||||||
|
|
||||||
|
|
||||||
|
async def upload_file(ws, path: str, media_ref: str, next_id: int) -> int:
|
||||||
|
"""Drive media.upload.start -> binary chunks -> media.upload.end.
|
||||||
|
|
||||||
|
Returns the next free request id; raises on a non-ack terminal frame.
|
||||||
|
"""
|
||||||
|
data = open(path, "rb").read()
|
||||||
|
mime, _ = mimetypes.guess_type(path)
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"v": 1, "id": next_id, "type": "media.upload.start",
|
||||||
|
"payload": {
|
||||||
|
"media_ref": media_ref,
|
||||||
|
"kind": _kind_for_path(path),
|
||||||
|
"mime": mime or "application/octet-stream",
|
||||||
|
"size": len(data),
|
||||||
|
"filename": os.path.basename(path),
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
print(f" -> media.upload.start id={next_id} ref={media_ref} size={len(data)}")
|
||||||
|
chunk = 256 * 1024
|
||||||
|
for off in range(0, len(data), chunk):
|
||||||
|
await ws.send(data[off:off + chunk])
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"v": 1, "id": next_id + 1, "type": "media.upload.end",
|
||||||
|
"payload": {"media_ref": media_ref, "sha256": hashlib.sha256(data).hexdigest()},
|
||||||
|
}))
|
||||||
|
print(f" -> media.upload.end id={next_id + 1} ref={media_ref}")
|
||||||
|
while True:
|
||||||
|
raw = await asyncio.wait_for(ws.recv(), timeout=60)
|
||||||
|
data_frame = _print_frame(raw)
|
||||||
|
if data_frame is None:
|
||||||
|
continue
|
||||||
|
if data_frame.get("type") == "media.upload.ack":
|
||||||
|
if not data_frame["payload"].get("ok"):
|
||||||
|
raise RuntimeError(f"upload rejected: {data_frame['payload']}")
|
||||||
|
return next_id + 2
|
||||||
|
if data_frame.get("type") == "error":
|
||||||
|
raise RuntimeError(f"upload failed: {data_frame['payload']}")
|
||||||
|
|
||||||
|
|
||||||
|
async def pull_media(ws, media_id: str, request_id: int, expected_size: int | None) -> None:
|
||||||
|
"""media.pull -> binary frames -> media.pull.end; verifies the size."""
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"v": 1, "id": request_id, "type": "media.pull",
|
||||||
|
"payload": {"media_id": media_id},
|
||||||
|
}))
|
||||||
|
print(f" -> media.pull id={request_id} media_id={media_id}")
|
||||||
|
total = 0
|
||||||
|
while True:
|
||||||
|
raw = await asyncio.wait_for(ws.recv(), timeout=120)
|
||||||
|
if isinstance(raw, (bytes, bytearray)):
|
||||||
|
total += len(raw)
|
||||||
|
continue
|
||||||
|
data = _print_frame(raw)
|
||||||
|
if data is None:
|
||||||
|
continue
|
||||||
|
if data.get("type") == "media.pull.end":
|
||||||
|
if not data["payload"].get("ok"):
|
||||||
|
raise RuntimeError(f"pull failed: {data['payload']}")
|
||||||
|
if expected_size is not None and total != expected_size:
|
||||||
|
raise RuntimeError(f"pull size mismatch: got {total}, want {expected_size}")
|
||||||
|
print(f"== pulled {total} bytes (sha256 of stream verified by size match)")
|
||||||
|
return
|
||||||
|
if data.get("type") == "error":
|
||||||
|
raise RuntimeError(f"pull failed: {data['payload']}")
|
||||||
|
|
||||||
|
|
||||||
async def run(args) -> int:
|
async def run(args) -> int:
|
||||||
url = args.url
|
url = args.url
|
||||||
token = args.token
|
token = args.token
|
||||||
@@ -132,22 +224,38 @@ async def run(args) -> int:
|
|||||||
await ws.close()
|
await ws.close()
|
||||||
return 5
|
return 5
|
||||||
|
|
||||||
if not args.send:
|
if not args.send and not args.upload:
|
||||||
print("== paired OK (no --send; exiting)")
|
print("== paired OK (no --send/--upload; exiting)")
|
||||||
await ws.close()
|
await ws.close()
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
# M4: optional inbound upload before the turn.
|
||||||
|
media_refs: list[str] = []
|
||||||
|
next_id = 1
|
||||||
|
if args.upload:
|
||||||
|
media_ref = f"mu_probe_{uuid.uuid4().hex[:8]}"
|
||||||
|
try:
|
||||||
|
next_id = await upload_file(ws, args.upload, media_ref, next_id)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"!! upload failed: {e}")
|
||||||
|
await ws.close()
|
||||||
|
return 8
|
||||||
|
media_refs.append(media_ref)
|
||||||
|
|
||||||
# Drive a turn.
|
# Drive a turn.
|
||||||
msg_id = 1
|
msg_id = next_id
|
||||||
|
send_payload: dict = {"text": args.send or ""}
|
||||||
|
if media_refs:
|
||||||
|
send_payload["media_refs"] = media_refs
|
||||||
send_frame = {
|
send_frame = {
|
||||||
"v": 1,
|
"v": 1,
|
||||||
"id": msg_id,
|
"id": msg_id,
|
||||||
"type": "message.send",
|
"type": "message.send",
|
||||||
"chat_id": "android:default",
|
"chat_id": "android:default",
|
||||||
"payload": {"text": args.send},
|
"payload": send_payload,
|
||||||
}
|
}
|
||||||
await ws.send(json.dumps(send_frame))
|
await ws.send(json.dumps(send_frame))
|
||||||
print(f" -> message.send id={msg_id} text={args.send!r}")
|
print(f" -> message.send id={msg_id} text={args.send!r} media_refs={media_refs}")
|
||||||
|
|
||||||
deadline = time.time() + args.timeout
|
deadline = time.time() + args.timeout
|
||||||
got_final = False
|
got_final = False
|
||||||
@@ -164,6 +272,16 @@ async def run(args) -> int:
|
|||||||
continue
|
continue
|
||||||
ftype = data.get("type")
|
ftype = data.get("type")
|
||||||
payload = data.get("payload") or {}
|
payload = data.get("payload") or {}
|
||||||
|
# M4: fetch offered media live (outbound direction).
|
||||||
|
if ftype == "media.offer" and args.pull_offer and payload.get("media_id"):
|
||||||
|
try:
|
||||||
|
next_id = await pull_media(
|
||||||
|
ws, payload.get("media_id"), next_id, payload.get("size")
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"!! pull failed: {e}")
|
||||||
|
await ws.close()
|
||||||
|
return 9
|
||||||
# A standalone assistant `message` (non-streaming) is immediately final.
|
# A standalone assistant `message` (non-streaming) is immediately final.
|
||||||
if ftype == "message" and payload.get("role") == "assistant":
|
if ftype == "message" and payload.get("role") == "assistant":
|
||||||
got_final = True
|
got_final = True
|
||||||
@@ -189,6 +307,10 @@ def main() -> int:
|
|||||||
p.add_argument("--token", default=os.getenv("ANDROID_TOKEN", ""))
|
p.add_argument("--token", default=os.getenv("ANDROID_TOKEN", ""))
|
||||||
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
|
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
|
||||||
p.add_argument("--send", default="hello")
|
p.add_argument("--send", default="hello")
|
||||||
|
p.add_argument("--upload", default="",
|
||||||
|
help="M4: file to upload (chunked) and attach via media_refs")
|
||||||
|
p.add_argument("--pull-offer", action="store_true",
|
||||||
|
help="M4: pull any media.offer that arrives during the turn")
|
||||||
p.add_argument("--timeout", type=float, default=120.0)
|
p.add_argument("--timeout", type=float, default=120.0)
|
||||||
p.add_argument("--authfail", action="store_true",
|
p.add_argument("--authfail", action="store_true",
|
||||||
help="expect an auth rejection (wrong token)")
|
help="expect an auth rejection (wrong token)")
|
||||||
|
|||||||
@@ -39,6 +39,11 @@ logger = logging.getLogger(__name__)
|
|||||||
# How long a new socket may take to present its ``hello`` before we drop it.
|
# How long a new socket may take to present its ``hello`` before we drop it.
|
||||||
HELLO_TIMEOUT_S = 10.0
|
HELLO_TIMEOUT_S = 10.0
|
||||||
|
|
||||||
|
# Max time a single outbound send may block on a peer's full write buffer
|
||||||
|
# before we give up on that peer (so one stalled client can't starve the
|
||||||
|
# rest of the broadcast). The peer's own ping timeout reaps it afterwards.
|
||||||
|
SEND_TIMEOUT_S = 10.0
|
||||||
|
|
||||||
# Close codes (4000-4999 are reserved for applications).
|
# Close codes (4000-4999 are reserved for applications).
|
||||||
CLOSE_AUTH_FAILED = 4401
|
CLOSE_AUTH_FAILED = 4401
|
||||||
CLOSE_REPLACED = 4402
|
CLOSE_REPLACED = 4402
|
||||||
@@ -138,16 +143,20 @@ class WsServer:
|
|||||||
def device_ids(self) -> list:
|
def device_ids(self) -> list:
|
||||||
return list(self._connections.keys())
|
return list(self._connections.keys())
|
||||||
|
|
||||||
|
def connection(self, device_id: str) -> Optional[DeviceConnection]:
|
||||||
|
return self._connections.get(device_id)
|
||||||
|
|
||||||
# ── Outbound ──────────────────────────────────────────────────────────
|
# ── Outbound ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async def broadcast(self, frame: protocol.Frame) -> int:
|
async def broadcast(self, frame: protocol.Frame) -> int:
|
||||||
"""Send a frame to every connected device. Returns devices reached.
|
"""Send a frame to every connected device. Returns devices reached.
|
||||||
Best-effort: a dead socket is skipped (deregistered on its own close)."""
|
Best-effort: a dead or stalled socket is skipped (deregistered on its
|
||||||
|
own close) so one slow peer can't starve the others."""
|
||||||
data = frame.to_json()
|
data = frame.to_json()
|
||||||
sent = 0
|
sent = 0
|
||||||
for conn in list(self._connections.values()):
|
for conn in list(self._connections.values()):
|
||||||
try:
|
try:
|
||||||
await conn.ws.send(data)
|
await asyncio.wait_for(conn.ws.send(data), timeout=SEND_TIMEOUT_S)
|
||||||
sent += 1
|
sent += 1
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -159,7 +168,7 @@ class WsServer:
|
|||||||
if conn is None:
|
if conn is None:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
await conn.ws.send(frame.to_json())
|
await asyncio.wait_for(conn.ws.send(frame.to_json()), timeout=SEND_TIMEOUT_S)
|
||||||
return True
|
return True
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
@@ -262,14 +271,25 @@ class WsServer:
|
|||||||
current = self._connections.get(device_id)
|
current = self._connections.get(device_id)
|
||||||
if current is not None and current.ws is ws:
|
if current is not None and current.ws is ws:
|
||||||
self._connections.pop(device_id, None)
|
self._connections.pop(device_id, None)
|
||||||
|
# M4: drop in-flight upload temp files for this socket.
|
||||||
|
try:
|
||||||
|
self._adapter.on_connection_closed(device_id)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("android: connection cleanup failed for %s", device_id, exc_info=True)
|
||||||
logger.info("android: device disconnected: %s", device_id)
|
logger.info("android: device disconnected: %s", device_id)
|
||||||
|
|
||||||
# ── Inbound dispatch ──────────────────────────────────────────────────
|
# ── Inbound dispatch ──────────────────────────────────────────────────
|
||||||
|
|
||||||
async def _on_frame(self, ws: ServerConnection, device_id: str, raw: Any) -> None:
|
async def _on_frame(self, ws: ServerConnection, device_id: str, raw: Any) -> None:
|
||||||
|
# M4: binary frames are media upload chunks (raw bytes, no JSON
|
||||||
|
# envelope). Route them to the active upload session.
|
||||||
|
if isinstance(raw, (bytes, bytearray, memoryview)):
|
||||||
|
await self._adapter.on_media_chunk(device_id, bytes(raw))
|
||||||
|
return
|
||||||
|
|
||||||
frame = protocol.Frame.from_json(raw)
|
frame = protocol.Frame.from_json(raw)
|
||||||
if frame is None:
|
if frame is None:
|
||||||
return # malformed / unknown binary: ignore (forward-compat)
|
return # malformed JSON: ignore (forward-compat)
|
||||||
|
|
||||||
if frame.type == protocol.TYPE_PING:
|
if frame.type == protocol.TYPE_PING:
|
||||||
ts = frame.payload.get("ts")
|
ts = frame.payload.get("ts")
|
||||||
@@ -290,6 +310,12 @@ class WsServer:
|
|||||||
await self._adapter.on_search(frame, device_id)
|
await self._adapter.on_search(frame, device_id)
|
||||||
elif frame.type == protocol.TYPE_SYNC:
|
elif frame.type == protocol.TYPE_SYNC:
|
||||||
await self._adapter.on_sync(frame, device_id)
|
await self._adapter.on_sync(frame, device_id)
|
||||||
|
elif frame.type == protocol.TYPE_MEDIA_UPLOAD_START:
|
||||||
|
await self._adapter.on_media_upload_start(frame, device_id)
|
||||||
|
elif frame.type == protocol.TYPE_MEDIA_UPLOAD_END:
|
||||||
|
await self._adapter.on_media_upload_end(frame, device_id)
|
||||||
|
elif frame.type == protocol.TYPE_MEDIA_PULL:
|
||||||
|
await self._adapter.on_media_pull(frame, device_id)
|
||||||
elif frame.type == "fcm.register":
|
elif frame.type == "fcm.register":
|
||||||
fcm_token = frame.payload.get("fcm_token")
|
fcm_token = frame.payload.get("fcm_token")
|
||||||
ntfy_topic = frame.payload.get("ntfy_topic")
|
ntfy_topic = frame.payload.get("ntfy_topic")
|
||||||
|
|||||||
Reference in new issue
Block a user