ARIA/iris_x_hermes
1
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases 4 Wiki Activity

M7: polish + E2E + docs (layout pass, theming, states, e2e driver, schema, setup.md, security)

Browse Source
This commit is contained in:
ARIA committed 2026-08-20 12:00:13 +02:00
1 parent 0cc8b7aafe
commit bf6bf7e8bd
26 files changed
+2225 -327
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

gateway-plugin/adapter.py
+15
View File
@@ -691,6 +691,7 @@ class AndroidAdapter(BasePlatformAdapter):
self.max_upload_bytes = int(
extra.get("max_upload_bytes", DEFAULT_MAX_UPLOAD_BYTES)
)
self._gateway_status = protocol.STATUS_ONLINE
# Home channel: the core hook turns the env-seeded ``home_channel``
# dict into a HomeChannel dataclass on the config; config.yaml may
@@ -796,6 +797,10 @@ class AndroidAdapter(BasePlatformAdapter):
self._connected = False
return False
# M5: announce gateway health to connected clients (none yet at
# startup; the frame + plumbing exist for future transitions).
await self._ws_server.broadcast(protocol.status(self._gateway_status))
# M3: ensure the default (home) channel exists in the directory so the
# app's channel list and cron home delivery have a stable anchor.
try:
@@ -1542,6 +1547,12 @@ class AndroidAdapter(BasePlatformAdapter):
media_types=media_types,
)
await self.handle_message(event)
# M5: acknowledge the user message to the originating device (the
# app shows ✓✓) at the moment it is handed to the agent.
await self._ws_server.send_to(
device_id,
protocol.read_receipt(chat_id, message_id),
)
# ── M4: inbound media (app -> agent) ──────────────────────────────────
#
@@ -2023,6 +2034,10 @@ class AndroidAdapter(BasePlatformAdapter):
# ── hello.ack helpers ─────────────────────────────────────────────────
def gateway_status(self) -> str:
"""Current gateway health state (sent to each pairing connection)."""
return self._gateway_status
def server_caps(self) -> Dict[str, Any]:
"""Capability flags advertised in ``hello.ack`` (M4 surface)."""
return {
gateway-plugin/protocol.py
+32 -1
View File
@@ -12,7 +12,7 @@ Milestone M2: message.start/update/stop, reasoning (on message /
message.stop), tool.start/progress/end, commentary.
Milestone M3: channel.*, search, sync.
Milestone M4: media.*.
Milestone M5: notification, fcm.register, read.receipt.
Milestone M5: notification, fcm.register, read.receipt, status.
"""
import json
@@ -79,6 +79,10 @@ TYPE_MEDIA_PULL_END = "media.pull.end"
# Push / notifications (M5)
TYPE_NOTIFICATION = "notification"
TYPE_FCM_REGISTER = "fcm.register"
TYPE_READ_RECEIPT = "read.receipt"
# Gateway health (M5)
TYPE_STATUS = "status"
# ---------------------------------------------------------------------------
# Error codes (``error`` frame payload.code)
@@ -116,6 +120,14 @@ NOTIF_GENERIC = "generic"
# it decides whether to also show an in-app banner).
HIGH_PRIORITY_NOTIF_KINDS = frozenset({NOTIF_APPROVAL, NOTIF_CLARIFY, NOTIF_CRON})
# ---------------------------------------------------------------------------
# Gateway health states (``status`` frame payload.state)
# ---------------------------------------------------------------------------
STATUS_ONLINE = "online"
STATUS_RESTARTING = "restarting"
STATUS_DEGRADED = "degraded"
# ---------------------------------------------------------------------------
# Envelope
@@ -507,6 +519,25 @@ def fcm_register(
return Frame(type=TYPE_FCM_REGISTER, payload=payload)
def read_receipt(chat_id: str, message_id: str) -> Frame:
"""Ack to the originating device: the agent received and started
processing the user's message (the app shows ✓✓ on the user bubble)."""
return Frame(
type=TYPE_READ_RECEIPT,
chat_id=chat_id,
payload={"message_id": message_id},
)
# ---------------------------------------------------------------------------
# Gateway health frame (M5)
# ---------------------------------------------------------------------------
def status(state: str) -> Frame:
"""Gateway health state (``state`` is one of the ``STATUS_*`` constants)."""
return Frame(type=TYPE_STATUS, payload={"state": state})
# ---------------------------------------------------------------------------
# Media frames (M4)
# ---------------------------------------------------------------------------
gateway-plugin/tests/README.md
+70 -1
View File
@@ -4,4 +4,73 @@ Run via hermes's hermetic runner (never bare pytest)::
scripts/run_tests.sh tests/gateway/test_android.py
See ``docs/13-testing.md`` for the scenario list.
See ``docs/13-testing.md`` for the scenario list.
## WS probe (`ws_probe.py`)
Manual test-client harness: connects to the **real running gateway** and
drives a turn, printing every frame. Run with the hermes venv python
(needs `websockets`); the gateway must already be up::
hermes-agent/.venv/bin/python gateway-plugin/tests/ws_probe.py \
--token <ANDROID_TOKEN> --send "hello"
Beyond the base modes (`--send`, `--upload`, `--pull-offer`, `--sync`,
`--fcm-token`/`--fcm-reg`, `--authfail`, `--url`, `--token`, `--device`,
`--timeout`), the probe has assertion and request modes:
- `--assert-turn` — assert the turn produced `message.start` → ≥1
`message.update` → `message.stop` (scenario 2).
- `--assert-reasoning` — assert the final `message.stop` carries a
non-empty `reasoning` field (scenario 3).
- `--assert-tools` — assert ≥1 `tool.start` with a matching `tool.end`
(matched by `index`; scenario 4).
- `--assert-commentary` — assert ≥1 `commentary` frame (scenario 5).
- `--assert-read-receipt` — assert a `read.receipt` frame arrives after
the sent message (new M7 frame; requires `--send`). **SKIPs** (exit 0,
prints `== SKIP: …`) when the frame never arrives, e.g. against a
gateway that predates the M7 frames.
- `--assert-status` — assert a `status` frame is received (new M7 frame;
**SKIPs** when absent).
- `--search QUERY [--scope all|chat] [--chat-id C]` — send a `search`
frame (`{query, scope, limit}`) and assert ≥1 hit in `search.results`
(scenario 8). With `--send`, the turn is driven first, then the search.
- `--channel-create NAME` / `--channel-delete CHAT_ID` /
`--channel-list` — M3 channel directory management; create prints
`== channel created: <chat_id>` for scripting.
- `--watch CHAT_ID` — wait up to `--timeout` for a message to land in
`CHAT_ID` (cron delivery E2E, scenario 7).
- `--offer-grace S` — with `--pull-offer`, keep listening S seconds after
the final message for a `media.offer` (offers are emitted post-turn,
right after the final; default 15).
Exit codes: `0` ok (incl. SKIP for absent M7 frames), `2` connect fail,
`3` no hello.ack, `4` expected hello.ack, `5` authfail expected but
acked, `6` timeout, `7` no final message, `8` upload/sync fail,
`9` pull fail, `10` assert-turn fail, `11` assert-reasoning fail,
`12` assert-tools fail, `13` assert-commentary fail, `14` search fail
(error or zero hits), `15` channel.create/list fail, `16` channel.delete
fail, `17` watch timeout, `18` read.receipt arrived before the sent
message, `19` status frame with empty payload.
## E2E driver (`e2e.py`)
Runs the `docs/13-testing.md` §13.4 scenarios 1–12 automated-where-
possible against the live gateway, invoking `ws_probe.py` (and the
`hermes` CLI for cron) as subprocesses. Prints PASS / PARTIAL / SKIP /
FAIL per scenario plus a summary table; exits 0 if no FAIL, 1 otherwise::
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --skip 3,5,7
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --url ws://host:8790/ws
The token is read from `$ANDROID_TOKEN`, else `hermes-agent/.env`, else
`~/.hermes/.env`. The gateway must already be running (the driver never
starts or stops it). It is idempotent: channels/jobs it creates are
cleaned up even on failure, and leftover `e2e-*` channels/jobs from
earlier runs are removed at start.
Scenario notes: 3 (reasoning) and 5 (commentary) are model-dependent and
SKIP rather than FAIL when the current model does not emit them; 11
(push) and 12 (reconnect/sync) are PARTIAL by design — the WS leg is
automated, the device-notification / gateway-kill leg is manual.
gateway-plugin/tests/e2e.py
+373
View File
@@ -0,0 +1,373 @@
#!/usr/bin/env python3
"""E2E driver: docs/13-testing.md §13.4 scenarios 1-12 against the live gateway.
Drives ws_probe.py (and the hermes CLI for cron) as subprocesses. For each
scenario prints PASS / PARTIAL / SKIP / FAIL with a one-line reason, then a
summary table. Exit 0 if no FAIL, 1 otherwise.
Usage::
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --skip 3,5,7
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --url ws://host:8790/ws
The token is read from $ANDROID_TOKEN, else hermes-agent/.env, else
~/.hermes/.env. The gateway must already be running (this driver never
starts or stops it). Idempotent: channels/jobs it creates are cleaned up
even on failure, and leftover "e2e-*" channels/jobs from earlier runs are
removed at start.
Scenario notes:
3 (reasoning) and 5 (commentary) are model-dependent: they SKIP (not
FAIL) when the current model does not emit reasoning / commentary.
11 (push) and 12 (reconnect) are PARTIAL by design: the WS leg is
automated, the device-notification / gateway-kill leg is manual.
"""
import argparse
import os
import re
import struct
import subprocess
import sys
import uuid
import zlib
from pathlib import Path
HERE = Path(__file__).resolve().parent
REPO = HERE.parent.parent
PY = REPO / "hermes-agent" / ".venv" / "bin" / "python"
PROBE = HERE / "ws_probe.py"
HERMES = REPO / "hermes-agent" / ".venv" / "bin" / "hermes"
DEFAULT_URL = "ws://127.0.0.1:8790/ws"
PASS, PARTIAL, SKIP, FAIL = "PASS", "PARTIAL", "SKIP", "FAIL"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def find_token(cli_token: str) -> str:
if cli_token:
return cli_token
env = os.getenv("ANDROID_TOKEN")
if env:
return env
for p in (REPO / "hermes-agent" / ".env", Path.home() / ".hermes" / ".env"):
try:
for line in p.read_text().splitlines():
line = line.strip()
if line.startswith("ANDROID_TOKEN="):
return line.split("=", 1)[1].strip().strip('"').strip("'")
except OSError:
pass
return ""
def run_probe(env, url, token, *args, timeout=300):
cmd = [str(PY), str(PROBE), "--url", url, "--token", token, *args]
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
return p.returncode, p.stdout, p.stderr
def run_hermes(env, *args, timeout=120):
cmd = [str(HERMES), *args]
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
return p.returncode, p.stdout, p.stderr
def write_png(path: Path, color, size: int = 200) -> None:
"""Write a solid-color RGB PNG using only the stdlib (no PIL needed)."""
raw = b"".join(b"\x00" + bytes(color) * size for _ in range(size))
def chunk(tag: bytes, data: bytes) -> bytes:
return (struct.pack(">I", len(data)) + tag + data
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF))
ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)
path.write_bytes(
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", ihdr)
+ chunk(b"IDAT", zlib.compress(raw))
+ chunk(b"IEND", b"")
)
def parse_created_chat_id(out: str) -> str | None:
m = re.search(r"== channel created: (\S+)", out)
return m.group(1) if m else None
def sweep_leftovers(env, url, token) -> None:
"""Remove e2e-* channels / cron jobs left behind by earlier runs."""
rc, out, _ = run_probe(env, url, token, "--channel-list")
if rc == 0:
for m in re.finditer(r"== channel: (\S+) name='(e2e-[^']*)'", out):
chat_id, name = m.group(1), m.group(2)
print(f" cleanup: removing leftover channel {chat_id} ({name})")
run_probe(env, url, token, "--channel-delete", chat_id)
rc, out, _ = run_hermes(env, "cron", "list")
if rc == 0:
for m in re.finditer(
r"(\S+) \[(?:active|paused)\]\s*\n\s*Name:\s+(e2e-cron-[^ \n]*)", out
):
job_id, name = m.group(1), m.group(2)
print(f" cleanup: removing leftover cron job {job_id} ({name})")
run_hermes(env, "cron", "remove", job_id)
# ---------------------------------------------------------------------------
# Scenarios (docs/13-testing.md §13.4)
# ---------------------------------------------------------------------------
def s1_pair(env, url, token):
rc, _, _ = run_probe(env, url, "definitely-wrong-token", "--authfail", "--send", "")
if rc != 0:
return FAIL, f"wrong token was not rejected (rc={rc})"
rc, _, _ = run_probe(env, url, token, "--send", "")
if rc != 0:
return FAIL, f"valid token did not pair (rc={rc})"
return PASS, "wrong token rejected; hello.ack on valid token"
def s2_text(env, url, token):
prompt = "Write a short poem about the ocean, at least 8 lines"
rc, _, _ = run_probe(env, url, token, "--send", prompt,
"--assert-turn", "--timeout", "120")
if rc == 0:
return PASS, "message.start -> >=1 message.update -> message.stop"
if rc == 10:
return FAIL, "no ordered start/update/stop segment"
return FAIL, f"probe rc={rc}"
def s3_reasoning(env, url, token):
prompt = "Work out step by step: what is 17 * 23? Show your reasoning."
rc, _, _ = run_probe(env, url, token, "--send", prompt,
"--assert-reasoning", "--timeout", "120")
if rc == 0:
return PASS, "final message.stop carries non-empty reasoning"
if rc == 11:
return SKIP, "model returned no reasoning (model-dependent)"
return FAIL, f"probe rc={rc}"
def s4_tools(env, url, token):
prompt = ("List the files in your current working directory using your "
"shell tool, then tell me how many there are")
rc, _, _ = run_probe(env, url, token, "--send", prompt,
"--assert-tools", "--timeout", "150")
if rc == 0:
return PASS, "tool.start with a matching tool.end"
if rc == 12:
return FAIL, "no tool.start/tool.end pair"
return FAIL, f"probe rc={rc}"
def s5_commentary(env, url, token):
prompt = ("Research task: (1) use your shell tool to list the top-level "
"directories in /tmp, (2) report your findings so far, "
"(3) use your shell tool to count files in /tmp, "
"(4) report those findings too, (5) give a final summary of both")
rc, _, _ = run_probe(env, url, token, "--send", prompt,
"--assert-commentary", "--timeout", "150")
if rc == 0:
return PASS, "commentary frame observed"
if rc == 13:
return SKIP, "no commentary (model/agent-dependent per M2)"
return FAIL, f"probe rc={rc}"
def s6_channels(env, url, token):
name = f"e2e-chan-{uuid.uuid4().hex[:6]}"
rc, out, _ = run_probe(env, url, token, "--channel-create", name)
if rc != 0:
return FAIL, f"channel.create failed (rc={rc})"
chat_id = parse_created_chat_id(out)
if not chat_id:
return FAIL, "channel.created received but chat_id not parseable"
rc, _, _ = run_probe(env, url, token, "--channel-delete", chat_id)
if rc != 0:
run_probe(env, url, token, "--channel-delete", chat_id) # best-effort
return FAIL, f"channel.delete failed (rc={rc})"
return PASS, f"created {chat_id} + deleted (cleanup)"
def s7_cron(env, url, token):
chan_name = f"e2e-cron-chan-{uuid.uuid4().hex[:6]}"
rc, out, _ = run_probe(env, url, token, "--channel-create", chan_name)
if rc != 0:
return SKIP, f"could not create cron target channel (rc={rc})"
chat_id = parse_created_chat_id(out)
if not chat_id:
return FAIL, "channel.created received but chat_id not parseable"
job_name = f"e2e-cron-{uuid.uuid4().hex[:6]}"
deliver = f"android:{chat_id}"
rc, out, err = run_hermes(
env, "cron", "create", "1m",
"Reply with exactly: e2e cron delivery OK",
"--deliver", deliver, "--name", job_name,
)
job_id = None
if rc == 0:
m = re.search(r"Created job: (\S+)", out)
job_id = m.group(1) if m else None
try:
if rc != 0:
return SKIP, f"hermes cron create failed: {(err or out).strip()[:120]}"
rc, out, _ = run_probe(env, url, token, "--watch", chat_id,
"--timeout", "330", timeout=400)
if rc == 0:
return PASS, f"one-shot cron job fired; message landed in {chat_id}"
return FAIL, f"no message in {chat_id} within 330s (probe rc={rc})"
finally:
if job_id:
run_hermes(env, "cron", "remove", job_id)
else:
# create succeeded but the id was not parseable: find by name.
_, list_out, _ = run_hermes(env, "cron", "list")
m = re.search(r"(\S+) \[active\]\s*\n\s*Name:\s+" + re.escape(job_name),
list_out)
if m:
run_hermes(env, "cron", "remove", m.group(1))
run_probe(env, url, token, "--channel-delete", chat_id)
def s8_search(env, url, token):
marker = f"e2emarker{uuid.uuid4().hex[:8]}"
rc, _, _ = run_probe(env, url, token, "--send",
f"Remember this marker phrase: {marker}. "
"Just acknowledge it briefly.",
"--timeout", "120")
if rc != 0:
return FAIL, f"setup message failed (rc={rc})"
rc, _, _ = run_probe(env, url, token, "--send", "", "--search", marker)
if rc == 0:
return PASS, f"search for {marker!r} returned >=1 hit"
if rc == 14:
return FAIL, f"search for {marker!r} returned 0 hits"
return FAIL, f"probe rc={rc}"
def s9_media_in(env, url, token):
png = Path(f"/tmp/e2e_in_{uuid.uuid4().hex[:6]}.png")
write_png(png, (30, 120, 220))
try:
rc, _, _ = run_probe(env, url, token, "--upload", str(png),
"--send", "describe this image briefly",
"--timeout", "120")
if rc == 0:
return PASS, "upload + vision reply (final message)"
if rc == 8:
return FAIL, "media upload failed"
if rc == 7:
return FAIL, "no final message after upload"
return FAIL, f"probe rc={rc}"
finally:
png.unlink(missing_ok=True)
def s10_media_out(env, url, token):
prompt = ("Create a 100x100 orange square PNG in /tmp with your tools. "
"In your final reply, include the MEDIA:/absolute/path tag for "
"that file so it is delivered to me.")
rc, out, _ = run_probe(env, url, token, "--send", prompt,
"--pull-offer", "--timeout", "150")
m = re.search(r"== pulled (\d+) bytes", out)
if rc == 0 and m and int(m.group(1)) > 0:
return PASS, f"media.offer pulled ({m.group(1)} bytes)"
if rc == 9:
return FAIL, "media pull failed"
return FAIL, "no media.offer pulled (agent did not deliver an image)"
def s11_push(env, url, token):
rc, out, _ = run_probe(env, url, token, "--fcm-token", "test-token-123",
"--fcm-reg", "--send", "")
if rc != 0:
return FAIL, f"probe rc={rc}"
if "<- error" in out:
return FAIL, "error frame after fcm.register"
return PARTIAL, ("fcm.register accepted (no error frame); "
"device-notification leg is manual")
def s12_sync(env, url, token):
rc, out, _ = run_probe(env, url, token, "--sync", "0")
if rc == 0 and "sync done" in out:
return PARTIAL, ("sync replay + sync.done verified; "
"gateway-kill/restart leg is manual")
if rc == 8:
return FAIL, "sync failed"
return FAIL, f"probe rc={rc}"
SCENARIOS = [
(1, "pair", s1_pair),
(2, "text round-trip", s2_text),
(3, "reasoning", s3_reasoning),
(4, "tools", s4_tools),
(5, "commentary", s5_commentary),
(6, "channels", s6_channels),
(7, "cron delivery", s7_cron),
(8, "search", s8_search),
(9, "media in", s9_media_in),
(10, "media out", s10_media_out),
(11, "push", s11_push),
(12, "reconnect/sync", s12_sync),
]
def main() -> int:
p = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
p.add_argument("--url", default=os.getenv("ANDROID_WS_URL", DEFAULT_URL))
p.add_argument("--token", default="")
p.add_argument("--skip", default="",
help="comma-separated scenario numbers to skip (e.g. 3,5,7)")
args = p.parse_args()
token = find_token(args.token)
if not token:
print("!! ANDROID_TOKEN not found (env, hermes-agent/.env, or ~/.hermes/.env)")
return 1
skip = {int(x) for x in args.skip.split(",") if x.strip()}
env = dict(os.environ)
env["ANDROID_TOKEN"] = token
print(f"== e2e: url={args.url} token={token[:6]}…")
sweep_leftovers(env, args.url, token)
results = []
for num, name, fn in SCENARIOS:
if num in skip:
results.append((num, name, SKIP, "skipped by --skip"))
print(f"[{num:2d}] {name:<18} {SKIP:<7} skipped by --skip")
continue
print(f"[{num:2d}] {name:<18} running…", flush=True)
try:
status, reason = fn(env, args.url, token)
except Exception as e:
status, reason = FAIL, f"driver error: {e}"
results.append((num, name, status, reason))
print(f"[{num:2d}] {name:<18} {status:<7} {reason}")
print()
print("=" * 78)
print(f"{'#':<3} {'scenario':<18} {'status':<8} reason")
print("-" * 78)
for num, name, status, reason in results:
print(f"{num:<3} {name:<18} {status:<8} {reason}")
print("-" * 78)
counts = {s: sum(1 for r in results if r[2] == s)
for s in (PASS, PARTIAL, SKIP, FAIL)}
print(f"total: {len(results)} PASS={counts[PASS]} PARTIAL={counts[PARTIAL]} "
f"SKIP={counts[SKIP]} FAIL={counts[FAIL]}")
return 1 if counts[FAIL] else 0
if __name__ == "__main__":
sys.exit(main())
gateway-plugin/tests/ws_probe.py
+437 -58
View File
@@ -18,14 +18,64 @@ Options:
--send TEXT send this message after pairing (default: "hello")
--upload F M4: upload F (chunked media.upload) and attach it to the
message.send via media_refs
--pull-offer M4: when a media.offer arrives during the turn, pull the
media (chunked) and verify the byte count
--pull-offer M4: when a media.offer arrives during the turn, pull the
media (chunked) and verify the byte count. Offers are
emitted right AFTER the final message (MEDIA: tag
extraction runs post-turn), so after the final the probe
keeps listening for --offer-grace seconds for one.
--sync C M5: after pairing, send sync {cursor: C} and print the
replay + sync.done (no turn is driven)
replay + sync.done (no turn is driven)
--fcm-token M5: attach this FCM token to the hello payload
--fcm-reg M5: after pairing, send fcm.register with --fcm-token
--timeout S seconds to wait for the final reply (default 120)
--authfail expect an auth rejection (wrong token) and exit 0 on it
Assertion modes (checked after the turn; see exit codes below):
--assert-turn M2: the turn produced message.start -> >=1
message.update -> message.stop (scenario 2)
--assert-reasoning M2: the final message.stop carries a non-empty
reasoning field (scenario 3)
--assert-tools M2: >=1 tool.start with a matching tool.end
(matched by index; scenario 4)
--assert-commentary M2: >=1 commentary frame (scenario 5)
--assert-read-receipt M7: a read.receipt frame arrives after the sent
message. SKIPs (exit 0) when the frame never
arrives (old gateway without the M7 frame).
--assert-status M7: a status frame is received. SKIPs (exit 0)
when the frame never arrives.
Request modes (no turn driven unless --send/--upload also given):
--search Q [--scope all|chat] [--chat-id C]
M3: send search {query, scope, limit} and assert >=1 hit
in search.results (scenario 8). With --send, the turn is
driven first, then the search runs.
--channel-create NAME M3: send channel.create, print the new chat_id
("== channel created: <chat_id>"), exit
--channel-delete CHAT M3: send channel.delete, assert channel.deleted
--channel-list M3: send channel.list, print the directory
--watch CHAT_ID wait up to --timeout for a message to land in
CHAT_ID (cron delivery E2E, scenario 7)
Exit codes:
0 ok (incl. SKIP for absent M7 frames)
2 connect failed
3 no hello.ack
4 expected hello.ack, got something else
5 --authfail but the token was accepted
6 timeout waiting for the final message
7 no final assistant message
8 upload/sync failed
9 media pull failed
10 --assert-turn failed (no ordered start/update/stop segment)
11 --assert-reasoning failed (final has no non-empty reasoning)
12 --assert-tools failed (no tool.start with a matching tool.end)
13 --assert-commentary failed (no commentary frame)
14 --search failed (error or zero hits)
15 --channel-create / --channel-list failed
16 --channel-delete failed
17 --watch timed out (no message landed in the channel)
18 --assert-read-receipt failed (frame arrived before the sent message)
19 --assert-status failed (status frame arrived with an empty payload)
"""
import argparse
@@ -105,6 +155,19 @@ def _print_frame(raw):
extra = f" cursor={payload.get('cursor')}"
elif ftype == "sync.done":
extra = f" cursor={payload.get('cursor')}"
elif ftype == "search.results":
hits = payload.get("hits") or []
extra = f" query={payload.get('query')!r} scope={payload.get('scope')} hits={len(hits)}"
elif ftype == "channel.created":
extra = f" chat_id={payload.get('chat_id')} name={payload.get('name')!r}"
elif ftype == "channel.deleted":
extra = f" chat_id={payload.get('chat_id')}"
elif ftype == "channel.list":
extra = f" channels={len(payload.get('channels') or [])}"
elif ftype == "read.receipt":
extra = f" payload={ {k: payload[k] for k in list(payload)[:4]} }"
elif ftype == "status":
extra = f" payload={ {k: payload[k] for k in list(payload)[:4]} }"
scope = f" chat={chat}" if chat else ""
idpart = f" id={fid}" if fid is not None else ""
print(f" <- {ftype}{idpart}{scope}{extra}")
@@ -189,6 +252,228 @@ async def pull_media(ws, media_id: str, request_id: int, expected_size: int | No
raise RuntimeError(f"pull failed: {data['payload']}")
class _TurnState:
"""Assertion-relevant facts collected while driving a turn."""
def __init__(self):
self.seq: list[tuple[str, str | None]] = [] # (type, message_id)
self.tool_starts: set[int] = set()
self.tool_ends: set[int] = set()
self.commentary = 0
self.final_stop_reasoning: str | None = None
self.final_message_reasoning: str | None = None
self.user_echo_seen = False
self.read_receipt: bool | None = None # None = never arrived
self.status_seen = False
self.status_empty = False
self.pulled = False
def track(self, ftype: str, payload: dict) -> None:
if ftype in ("message.start", "message.update", "message.stop"):
self.seq.append((ftype, payload.get("message_id")))
if ftype == "message.stop":
r = payload.get("reasoning")
if isinstance(r, str) and r.strip():
self.final_stop_reasoning = r
if ftype == "message" and payload.get("role") == "assistant":
r = payload.get("reasoning")
if isinstance(r, str) and r.strip():
self.final_message_reasoning = r
if ftype == "message" and payload.get("role") == "user":
self.user_echo_seen = True
if ftype == "tool.start" and isinstance(payload.get("index"), int):
self.tool_starts.add(payload["index"])
if ftype == "tool.end" and isinstance(payload.get("index"), int):
self.tool_ends.add(payload["index"])
if ftype == "commentary":
self.commentary += 1
if ftype == "read.receipt":
self.read_receipt = self.user_echo_seen
if ftype == "status":
self.status_seen = True
if not payload:
self.status_empty = True
def _evaluate_assertions(args, st: _TurnState) -> list[tuple[int, bool, str]]:
"""Evaluate the enabled assertion modes. Returns (exit_code, ok, message)
per failed-or-passed assertion; SKIPs are printed here and not returned."""
results: list[tuple[int, bool, str]] = []
if args.assert_turn:
ok = False
for mid in {m for _, m in st.seq if m is not None}:
events = [t for t, m in st.seq if m == mid]
if "message.start" in events and "message.stop" in events:
i_start = events.index("message.start")
i_stop = events.index("message.stop")
if any(i_start < i < i_stop
for i, e in enumerate(events) if e == "message.update"):
ok = True
break
results.append((10, ok,
"assert-turn: no message.start -> >=1 message.update -> message.stop"))
if args.assert_reasoning:
reasoning = st.final_stop_reasoning or st.final_message_reasoning
results.append((11, bool(reasoning),
"assert-reasoning: final message has no non-empty reasoning"))
if args.assert_tools:
ok = bool(st.tool_starts) and bool(st.tool_starts & st.tool_ends)
results.append((12, ok,
"assert-tools: no tool.start with a matching tool.end"))
if args.assert_commentary:
results.append((13, st.commentary >= 1,
"assert-commentary: no commentary frame"))
if args.assert_read_receipt:
if st.read_receipt is None:
print("== SKIP: no read.receipt frame (M7 frame not live on this gateway)")
elif not st.read_receipt:
results.append((18, False,
"assert-read-receipt: read.receipt arrived before the sent message"))
if args.assert_status:
if not st.status_seen:
print("== SKIP: no status frame (M7 frame not live on this gateway)")
elif st.status_empty:
results.append((19, False,
"assert-status: status frame arrived with an empty payload"))
return results
async def _recv_frames(ws, timeout: float):
"""Yield parsed frames (dicts) until *timeout* seconds elapse."""
deadline = time.time() + timeout
while time.time() < deadline:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=deadline - time.time())
except asyncio.TimeoutError:
return
if isinstance(raw, (bytes, bytearray)):
continue
data = _print_frame(raw)
if data is not None:
yield data
async def _search_mode(ws, args, next_id: int) -> int:
"""M3: send a search frame, wait for search.results, assert >=1 hit."""
req_id = next_id
payload = {"query": args.search, "scope": args.scope, "limit": 20}
if args.scope == "chat":
payload["chat_id"] = args.chat_id
await ws.send(json.dumps({"v": 1, "id": req_id, "type": "search", "payload": payload}))
print(f" -> search id={req_id} query={args.search!r} scope={args.scope}")
async for data in _recv_frames(ws, timeout=30):
if data.get("type") == "search.results" and data.get("id") == req_id:
hits = (data.get("payload") or {}).get("hits") or []
print(f"== search: {len(hits)} hit(s)")
for h in hits[:10]:
print(f" hit chat={h.get('chat_id')} role={h.get('role')} "
f"snippet={str(h.get('snippet'))[:100]!r}")
if hits:
return 0
print("!! search: no hits")
return 14
if data.get("type") == "error":
print(f"!! search failed: {data.get('payload')}")
return 14
print("!! search: no search.results within 30s")
return 14
async def _channel_create_mode(ws, args) -> int:
"""M3: channel.create -> channel.created; print the new chat_id."""
req_id = 1
await ws.send(json.dumps({
"v": 1, "id": req_id, "type": "channel.create",
"payload": {"name": args.channel_create},
}))
print(f" -> channel.create id={req_id} name={args.channel_create!r}")
async for data in _recv_frames(ws, timeout=30):
if data.get("type") == "channel.created" and data.get("id") == req_id:
chat_id = (data.get("payload") or {}).get("chat_id")
print(f"== channel created: {chat_id}")
await ws.close()
return 0
if data.get("type") == "error":
print(f"!! channel.create failed: {data.get('payload')}")
await ws.close()
return 15
print("!! channel.create: no channel.created within 30s")
await ws.close()
return 15
async def _channel_delete_mode(ws, args) -> int:
"""M3: channel.delete -> channel.deleted."""
req_id = 1
await ws.send(json.dumps({
"v": 1, "id": req_id, "type": "channel.delete",
"payload": {"chat_id": args.channel_delete},
}))
print(f" -> channel.delete id={req_id} chat_id={args.channel_delete!r}")
async for data in _recv_frames(ws, timeout=30):
if data.get("type") == "channel.deleted" and data.get("id") == req_id:
print(f"== channel deleted: {args.channel_delete}")
await ws.close()
return 0
if data.get("type") == "error":
print(f"!! channel.delete failed: {data.get('payload')}")
await ws.close()
return 16
print("!! channel.delete: no channel.deleted within 30s")
await ws.close()
return 16
async def _channel_list_mode(ws, args) -> int:
"""M3: channel.list -> print the directory."""
req_id = 1
await ws.send(json.dumps({"v": 1, "id": req_id, "type": "channel.list", "payload": {}}))
print(" -> channel.list")
async for data in _recv_frames(ws, timeout=30):
if data.get("type") == "channel.list" and data.get("id") == req_id:
for c in (data.get("payload") or {}).get("channels") or []:
print(f"== channel: {c.get('chat_id')} name={c.get('name')!r} "
f"default={bool(c.get('is_default'))}")
await ws.close()
return 0
if data.get("type") == "error":
print(f"!! channel.list failed: {data.get('payload')}")
await ws.close()
return 15
print("!! channel.list: no response within 30s")
await ws.close()
return 15
async def _watch_mode(ws, args) -> int:
"""Wait up to --timeout for a message to land in args.watch (cron E2E)."""
print(f"== watching {args.watch} for a message (timeout {args.timeout:.0f}s)")
deadline = time.time() + args.timeout
while time.time() < deadline:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=deadline - time.time())
except asyncio.TimeoutError:
print(f"!! timeout after {args.timeout:.0f}s watching {args.watch}")
await ws.close()
return 17
if isinstance(raw, (bytes, bytearray)):
continue
data = _print_frame(raw)
if data is None:
continue
if data.get("chat_id") != args.watch:
continue
ftype = data.get("type")
payload = data.get("payload") or {}
if ftype == "message" and payload.get("role") in ("assistant", "cron"):
print(f"== message landed in {args.watch}: {str(payload.get('text'))[:120]!r}")
await ws.close()
return 0
print(f"!! no message landed in {args.watch}")
await ws.close()
return 17
async def run(args) -> int:
url = args.url
token = args.token
@@ -266,8 +551,18 @@ async def run(args) -> int:
await ws.close()
return 8
if not args.send and not args.upload:
print("== paired OK (no --send/--upload; exiting)")
# M3: request modes (no turn driven).
if args.channel_create:
return await _channel_create_mode(ws, args)
if args.channel_delete:
return await _channel_delete_mode(ws, args)
if args.channel_list:
return await _channel_list_mode(ws, args)
if args.watch:
return await _watch_mode(ws, args)
if not args.send and not args.upload and not args.search:
print("== paired OK (no --send/--upload/--search; exiting)")
await ws.close()
return 0
@@ -284,63 +579,114 @@ async def run(args) -> int:
return 8
media_refs.append(media_ref)
# Drive a turn.
msg_id = next_id
send_payload: dict = {"text": args.send or ""}
if media_refs:
send_payload["media_refs"] = media_refs
send_frame = {
"v": 1,
"id": msg_id,
"type": "message.send",
"chat_id": "android:default",
"payload": send_payload,
}
await ws.send(json.dumps(send_frame))
print(f" -> message.send id={msg_id} text={args.send!r} media_refs={media_refs}")
deadline = time.time() + args.timeout
# Drive a turn (if --send or --upload).
st = _TurnState()
got_final = False
seen_final_frame = False
while time.time() < deadline:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=deadline - time.time())
except asyncio.TimeoutError:
print(f"!! timeout after {args.timeout}s waiting for final message")
await ws.close()
return 6
data = _print_frame(raw)
if data is None:
continue
ftype = data.get("type")
payload = data.get("payload") or {}
# M4: fetch offered media live (outbound direction).
if ftype == "media.offer" and args.pull_offer and payload.get("media_id"):
if args.send or args.upload:
msg_id = next_id
send_payload: dict = {"text": args.send or ""}
if media_refs:
send_payload["media_refs"] = media_refs
send_frame = {
"v": 1,
"id": msg_id,
"type": "message.send",
"chat_id": "android:default",
"payload": send_payload,
}
await ws.send(json.dumps(send_frame))
print(f" -> message.send id={msg_id} text={args.send!r} media_refs={media_refs}")
deadline = time.time() + args.timeout
seen_final_frame = False
while time.time() < deadline:
try:
next_id = await pull_media(
ws, payload.get("media_id"), next_id, payload.get("size")
)
except Exception as e:
print(f"!! pull failed: {e}")
raw = await asyncio.wait_for(ws.recv(), timeout=deadline - time.time())
except asyncio.TimeoutError:
print(f"!! timeout after {args.timeout}s waiting for final message")
await ws.close()
return 9
# A standalone assistant `message` (non-streaming) is immediately final.
if ftype == "message" and payload.get("role") == "assistant":
got_final = True
break
# A `message.stop` finalizes a streaming segment; the turn is done once
# typing stops afterwards (multi-segment turns have several stops).
if ftype == "message.stop":
seen_final_frame = True
if ftype == "typing" and payload.get("on") is False and seen_final_frame:
got_final = True
break
await ws.close()
if got_final:
return 6
data = _print_frame(raw)
if data is None:
continue
ftype = data.get("type")
payload = data.get("payload") or {}
st.track(ftype, payload)
# M4: fetch offered media live (outbound direction).
if ftype == "media.offer" and args.pull_offer and payload.get("media_id"):
try:
await pull_media(
ws, payload["media_id"], next_id, payload.get("size")
)
next_id += 1
st.pulled = True
except Exception as e:
print(f"!! pull failed: {e}")
await ws.close()
return 9
# A standalone assistant `message` (non-streaming) is immediately final.
if ftype == "message" and payload.get("role") == "assistant":
got_final = True
break
# A `message.stop` finalizes a streaming segment; the turn is done
# once typing stops afterwards (multi-segment turns have several
# stops).
if ftype == "message.stop":
seen_final_frame = True
if ftype == "typing" and payload.get("on") is False and seen_final_frame:
got_final = True
break
# M4: media offers are emitted right AFTER the final message (the
# MEDIA: tag is extracted post-turn); give them a grace window.
if got_final and args.pull_offer and not st.pulled:
grace_deadline = time.time() + args.offer_grace
while time.time() < grace_deadline:
try:
raw = await asyncio.wait_for(
ws.recv(), timeout=grace_deadline - time.time()
)
except asyncio.TimeoutError:
break
if isinstance(raw, (bytes, bytearray)):
continue
data = _print_frame(raw)
if data is None:
continue
if data.get("type") == "media.offer" and (data.get("payload") or {}).get("media_id"):
try:
await pull_media(
ws, data["payload"]["media_id"], next_id,
data["payload"].get("size"),
)
next_id += 1
st.pulled = True
except Exception as e:
print(f"!! pull failed: {e}")
await ws.close()
return 9
break
if not st.pulled:
print(f"== no media.offer within {args.offer_grace:.0f}s grace")
if not got_final:
print("!! no final assistant message")
await ws.close()
return 7
print("== final assistant message received")
return 0
print("!! no final assistant message")
return 7
# M3: optional search (standalone, or after the turn).
if args.search:
rc = await _search_mode(ws, args, next_id)
await ws.close()
return rc
await ws.close()
for code, ok, msg in _evaluate_assertions(args, st):
if not ok:
print(f"!! {msg}")
return code
return 0
def main() -> int:
@@ -362,9 +708,42 @@ def main() -> int:
p.add_argument("--timeout", type=float, default=120.0)
p.add_argument("--authfail", action="store_true",
help="expect an auth rejection (wrong token)")
p.add_argument("--assert-turn", action="store_true",
help="assert message.start -> >=1 message.update -> message.stop")
p.add_argument("--assert-reasoning", action="store_true",
help="assert the final message.stop carries non-empty reasoning")
p.add_argument("--assert-tools", action="store_true",
help="assert >=1 tool.start with a matching tool.end")
p.add_argument("--assert-commentary", action="store_true",
help="assert >=1 commentary frame")
p.add_argument("--assert-read-receipt", action="store_true",
help="assert a read.receipt arrives after the sent message "
"(SKIP if absent; M7)")
p.add_argument("--assert-status", action="store_true",
help="assert a status frame is received (SKIP if absent; M7)")
p.add_argument("--search", default="",
help="M3: send search {query, scope, limit}, assert >=1 hit")
p.add_argument("--scope", choices=("all", "chat"), default="all",
help="search scope (default all)")
p.add_argument("--chat-id", default="android:default",
help="chat_id for --scope chat (default android:default)")
p.add_argument("--channel-create", default="",
help="M3: create a channel, print its chat_id, exit")
p.add_argument("--channel-delete", default="",
help="M3: delete (archive) a channel, exit")
p.add_argument("--channel-list", action="store_true",
help="M3: list channels, exit")
p.add_argument("--watch", default="",
help="wait up to --timeout for a message to land in this chat_id")
p.add_argument("--offer-grace", type=float, default=15.0,
help="seconds to wait for a media.offer after the final "
"message when --pull-offer (default 15)")
args = p.parse_args()
if not args.token and not args.authfail:
p.error("--token (or $ANDROID_TOKEN) is required")
if args.assert_read_receipt and not args.send:
p.error("--assert-read-receipt requires --send (the receipt must follow "
"the sent message)")
return asyncio.run(run(args))
gateway-plugin/ws_server.py
+83 -6
View File
@@ -11,7 +11,9 @@ Per-connection handler:
2. On success: register in the device registry (SQLite) + connection
registry (``device_id -> {ws, caps, fcm_token}``), send
``hello.ack {server_caps, sync_cursor, channels[]}``.
3. Loop: decode frames, dispatch to adapter inbound handlers.
3. Loop: decode frames, dispatch to adapter inbound handlers. Inbound JSON
frames are rate-limited per connection (token bucket, ``INBOUND_RATE_PER_S``
/ ``INBOUND_BURST``); binary media-upload chunks are exempt.
4. On close: deregister.
Routing: ``broadcast(frame)`` sends to ALL connected devices (single-user
@@ -44,12 +46,46 @@ HELLO_TIMEOUT_S = 10.0
# rest of the broadcast). The peer's own ping timeout reaps it afterwards.
SEND_TIMEOUT_S = 10.0
# Inbound JSON control-frame rate limit (per connection, token bucket).
# A legitimate app sends pings + occasional user-initiated requests — far
# below 20/s sustained. Binary media-upload chunks are EXEMPT (see
# ``_on_frame``): a 100 MB upload is 400 x 256 KiB frames in a tight loop
# and would exhaust any sane bucket; uploads are bounded instead by the
# per-frame ``max_size`` and the per-upload total cap (``media.py``).
INBOUND_RATE_PER_S = 20.0
INBOUND_BURST = 40
# Close codes (4000-4999 are reserved for applications).
CLOSE_AUTH_FAILED = 4401
CLOSE_REPLACED = 4402
CLOSE_RATE_LIMITED = 4403
CLOSE_SHUTDOWN = 1001
class _TokenBucket:
"""Minimal token bucket (stdlib only). One instance per connection."""
__slots__ = ("rate", "burst", "tokens", "updated_at")
def __init__(self, rate: float, burst: int):
self.rate = rate
self.burst = burst
self.tokens = float(burst)
self.updated_at = time.monotonic()
def consume(self) -> bool:
"""Try to take one token. Refills at ``rate``/s up to ``burst``."""
now = time.monotonic()
elapsed = now - self.updated_at
if elapsed > 0:
self.tokens = min(self.burst, self.tokens + elapsed * self.rate)
self.updated_at = now
if self.tokens >= 1.0:
self.tokens -= 1.0
return True
return False
@dataclass
class DeviceConnection:
"""One live, authenticated device socket."""
@@ -61,6 +97,9 @@ class DeviceConnection:
fcm_token: Optional[str] = None
ntfy_topic: Optional[str] = None
connected_at: float = field(default_factory=time.time)
rate_bucket: _TokenBucket = field(
default_factory=lambda: _TokenBucket(INBOUND_RATE_PER_S, INBOUND_BURST)
)
class WsServer:
@@ -254,6 +293,9 @@ class WsServer:
)
try:
await ws.send(ack.to_json())
# M7: tell late-joining clients the current gateway health state
# (the startup broadcast only reaches clients already connected).
await ws.send(protocol.status(self._adapter.gateway_status()).to_json())
except Exception:
return
logger.info("android: device paired: %s (%s)", device_name, device_id)
@@ -261,7 +303,11 @@ class WsServer:
# 3. frame loop ------------------------------------------------------
try:
async for raw in ws:
await self._on_frame(ws, device_id, raw)
# ``_on_frame`` returns False once it has closed the socket
# (rate limit); stop draining the buffered frames so a
# flood doesn't re-trigger the error+close per frame.
if not await self._on_frame(ws, device_id, raw):
break
except ConnectionClosed:
pass
except Exception:
@@ -280,16 +326,38 @@ class WsServer:
# ── Inbound dispatch ──────────────────────────────────────────────────
async def _on_frame(self, ws: ServerConnection, device_id: str, raw: Any) -> None:
async def _on_frame(self, ws: ServerConnection, device_id: str, raw: Any) -> bool:
"""Dispatch one inbound frame. Returns False once the socket has been
closed (rate limit) so the caller stops draining buffered frames."""
# M4: binary frames are media upload chunks (raw bytes, no JSON
# envelope). Route them to the active upload session.
# envelope). Route them to the active upload session. They are
# EXEMPT from the inbound rate limit: a 100 MB upload is 400 x
# 256 KiB frames in a tight loop, which would exhaust any sane
# frame bucket. Uploads are bounded instead by the per-frame
# ``max_size`` and the per-upload total cap (``media.py``).
if isinstance(raw, (bytes, bytearray, memoryview)):
await self._adapter.on_media_chunk(device_id, bytes(raw))
return
return True
# Inbound rate limit (JSON control frames only). On exceed: error +
# close, same pattern as auth rejection.
conn = self._connection_for(ws)
if conn is not None and not conn.rate_bucket.consume():
logger.warning(
"android: inbound rate limit exceeded for %s; closing", device_id
)
await self._send_quiet(
ws,
protocol.error(
protocol.ERR_RATE_LIMITED, "inbound frame rate limit exceeded"
),
)
await self._close_quiet(ws, CLOSE_RATE_LIMITED, "rate limited")
return False
frame = protocol.Frame.from_json(raw)
if frame is None:
return # malformed JSON: ignore (forward-compat)
return True # malformed JSON: ignore (forward-compat)
if frame.type == protocol.TYPE_PING:
ts = frame.payload.get("ts")
@@ -319,9 +387,18 @@ class WsServer:
elif frame.type == protocol.TYPE_FCM_REGISTER:
await self._adapter.on_fcm_register(frame, device_id)
# Unknown types are ignored (forward-compat).
return True
# ── Helpers ───────────────────────────────────────────────────────────
def _connection_for(self, ws: ServerConnection) -> Optional[DeviceConnection]:
"""The live registry entry for this exact socket (identity match, so
a replaced socket never consumes the new connection's bucket)."""
for conn in self._connections.values():
if conn.ws is ws:
return conn
return None
async def _send_quiet(self, ws: ServerConnection, frame: protocol.Frame) -> None:
try:
await ws.send(frame.to_json())