M4: media upload/download/playback (both directions)
Gateway plugin: - media.upload (chunked binary) -> size/sha256 verify + MIME re-sniff -> cache_*_from_bytes -> media.upload.ack - media.offer / media.pull (chunked) for agent-sent media, delivery-path security re-checked at pull time - send_* overrides mint media_id and emit media.offer - message.send media_refs resolve to cached inbound media - per-send + per-chunk timeouts so a stalled peer can't starve the rest App (Kotlin CMP): - Protocol: media frame types/payloads/builders - GatewayClient: binary session, uploadMedia (chunked + streaming sha256), pullMedia serialized via Mutex so concurrent offers don't interleave - ChatStore/IrisController: MediaItem, attachments, auto-pull on offer - Platform media: SAF picker, ExoPlayer (audio mini-player + video), image loader, FileProvider document open (Android); AWT-free desktop actuals - ChatScreen: attach button + chips, media rendering, keyboard dismiss on send UI polish: - preserve image aspect ratio (no stretching), cap dominant dimension - adjustResize so only chat content squeezes for the keyboard - clear focus (hide keyboard) on send Docs: media.upload.ack in 04-wire-protocol.md + frames.schema.json + 07-media.md; M4 marked complete in 14-milestones.md. Tests: 17-test tests/gateway/test_android.py suite passes.
This commit is contained in:
ARIA committed
1 parent
60296b33fe
commit
913ee91024
28 files changed
+2234
-53
No files matched your search
@@ -0,0 +1,218 @@
|
||||
package iris.platform
|
||||
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.graphics.BitmapFactory
|
||||
import android.net.Uri
|
||||
import android.provider.OpenableColumns
|
||||
import android.widget.Toast
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Slider
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableFloatStateOf
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.ImageBitmap
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.core.content.FileProvider
|
||||
import androidx.core.net.toUri
|
||||
import androidx.media3.common.MediaItem as M3MediaItem
|
||||
import androidx.media3.common.Player
|
||||
import androidx.media3.exoplayer.ExoPlayer
|
||||
import androidx.media3.ui.PlayerView
|
||||
import iris.data.MediaItem
|
||||
import iris.media.extForMime
|
||||
import iris.protocol.KIND_VIDEO
|
||||
import java.io.File
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
/** App context holder (set in MainActivity.onCreate). */
|
||||
object AndroidEnv {
|
||||
lateinit var context: Context
|
||||
}
|
||||
|
||||
@Composable
|
||||
actual fun MediaFilePicker(onPicked: (PickedFile?) -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val launcher = rememberLauncherForActivityResult(ActivityResultContracts.GetContent()) { uri ->
|
||||
if (uri == null) {
|
||||
onPicked(null)
|
||||
return@rememberLauncherForActivityResult
|
||||
}
|
||||
try {
|
||||
val mime = context.contentResolver.getType(uri) ?: "application/octet-stream"
|
||||
val name = queryDisplayName(context, uri) ?: "upload"
|
||||
val dir = File(context.cacheDir, "uploads").apply { mkdirs() }
|
||||
val target = File(dir, "${System.currentTimeMillis()}${extForMime(mime)}")
|
||||
context.contentResolver.openInputStream(uri)!!.use { input ->
|
||||
target.outputStream().use { output -> input.copyTo(output) }
|
||||
}
|
||||
onPicked(PickedFile(target.absolutePath, name, mime, target.length()))
|
||||
} catch (e: Exception) {
|
||||
onPicked(null)
|
||||
}
|
||||
}
|
||||
Box(modifier = Modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
|
||||
Button(onClick = { launcher.launch("*/*") }) { Text("Pick a file") }
|
||||
}
|
||||
}
|
||||
|
||||
private fun queryDisplayName(context: Context, uri: Uri): String? {
|
||||
context.contentResolver.query(uri, null, null, null, null)?.use { c ->
|
||||
val idx = c.getColumnIndex(OpenableColumns.DISPLAY_NAME)
|
||||
if (idx >= 0 && c.moveToFirst()) return c.getString(idx)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
actual fun mediaCacheBaseDir(): String =
|
||||
AndroidEnv.context.externalCacheDir?.absolutePath
|
||||
?: AndroidEnv.context.cacheDir.absolutePath
|
||||
|
||||
actual suspend fun loadMediaImage(path: String): ImageBitmap? = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
BitmapFactory.decodeFile(path)?.asImageBitmap()
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
actual fun MediaPlayerView(media: MediaItem, modifier: Modifier) {
|
||||
val path = media.localPath
|
||||
if (path == null) {
|
||||
Box(modifier = modifier, contentAlignment = Alignment.Center) {
|
||||
Row {
|
||||
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
|
||||
Spacer(modifier = Modifier.size(8.dp))
|
||||
Text("downloading…", fontSize = 12.sp)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if (media.kind == KIND_VIDEO) {
|
||||
VideoPlayerView(path, modifier)
|
||||
} else {
|
||||
AudioPlayerView(path, media.filename, modifier)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun VideoPlayerView(path: String, modifier: Modifier) {
|
||||
AndroidView(
|
||||
factory = { ctx ->
|
||||
PlayerView(ctx).apply {
|
||||
useController = true
|
||||
val player = ExoPlayer.Builder(ctx).build()
|
||||
player.setMediaItem(M3MediaItem.fromUri(path.toUri()))
|
||||
player.prepare()
|
||||
player.playWhenReady = true
|
||||
this.player = player
|
||||
}
|
||||
},
|
||||
update = {},
|
||||
onRelease = { view -> view.player?.release() },
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.height(220.dp),
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AudioPlayerView(path: String, filename: String, modifier: Modifier) {
|
||||
val context = LocalContext.current
|
||||
val player = remember {
|
||||
ExoPlayer.Builder(context).build().apply {
|
||||
setMediaItem(M3MediaItem.fromUri(path.toUri()))
|
||||
prepare()
|
||||
}
|
||||
}
|
||||
DisposableEffect(player) { onDispose { player.release() } }
|
||||
|
||||
var playing by remember { mutableStateOf(false) }
|
||||
var position by remember { mutableFloatStateOf(0f) }
|
||||
var duration by remember { mutableFloatStateOf(0f) }
|
||||
|
||||
LaunchedEffect(player) {
|
||||
while (true) {
|
||||
val d = player.duration
|
||||
duration = if (d > 0) d / 1000f else 0f
|
||||
position = player.currentPosition / 1000f
|
||||
playing = player.isPlaying
|
||||
kotlinx.coroutines.delay(500)
|
||||
}
|
||||
}
|
||||
|
||||
Row(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.padding(8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
IconButton(onClick = {
|
||||
if (playing) player.pause() else player.play()
|
||||
}) {
|
||||
Text(if (playing) "⏸" else "▶️", fontSize = 18.sp)
|
||||
}
|
||||
Column(modifier = Modifier.weight(1f).padding(horizontal = 8.dp)) {
|
||||
Text(filename, fontSize = 12.sp, maxLines = 1)
|
||||
Slider(
|
||||
value = if (duration > 0f) (position / duration).coerceIn(0f, 1f) else 0f,
|
||||
onValueChange = { frac ->
|
||||
player.seekTo((frac * duration * 1000).toLong())
|
||||
},
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
Text(
|
||||
"${fmt(position)} / ${fmt(duration)}",
|
||||
fontSize = 10.sp,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun fmt(seconds: Float): String {
|
||||
val s = seconds.toLong()
|
||||
return "%d:%02d".format(s / 60, s % 60)
|
||||
}
|
||||
|
||||
actual fun openDocument(path: String, mime: String) {
|
||||
val context = AndroidEnv.context
|
||||
try {
|
||||
val file = File(path)
|
||||
val uri = FileProvider.getUriForFile(context, "${context.packageName}.fileprovider", file)
|
||||
val intent = Intent(Intent.ACTION_VIEW).apply {
|
||||
setDataAndType(uri, mime)
|
||||
addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
} catch (e: Exception) {
|
||||
Toast.makeText(context, "No app to open ${mime}", Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user