#!/usr/bin/env python3 """WS test-client harness (docs/13-testing.md §13.2). Connects to the REAL running gateway and drives a turn, printing every frame. This is how we empirically confirm the exact frame shapes before / while building the Kotlin client. Usage:: hermes gateway & # with the iris plugin python gateway-plugin/tests/ws_probe.py --token \ --send "hello" Options: --url ws://host:port/ws (default ws://127.0.0.1:8790/ws) --token IRIS_TOKEN (default: $IRIS_TOKEN) --device device_id (default: probe-) --send TEXT send this message after pairing (default: "hello") --upload F M4: upload F (chunked media.upload) and attach it to the message.send via media_refs --pull-offer M4: when a media.offer arrives during the turn, pull the media (chunked) and verify the byte count. Offers are emitted right AFTER the final message (MEDIA: tag extraction runs post-turn), so after the final the probe keeps listening for --offer-grace seconds for one. --sync C M5: after pairing, send sync {cursor: C} and print the replay + sync.done (no turn is driven) --fcm-token M5: attach this FCM token to the hello payload --fcm-reg M5: after pairing, send fcm.register with --fcm-token --timeout S seconds to wait for the final reply (default 120) --authfail expect an auth rejection (wrong token) and exit 0 on it Assertion modes (checked after the turn; see exit codes below): --assert-turn M2: the turn produced message.start -> >=1 message.update -> message.stop (scenario 2) --assert-reasoning M2: the final message.stop carries a non-empty reasoning field (scenario 3) --assert-tools M2: >=1 tool.start with a matching tool.end (matched by index; scenario 4) --assert-commentary M2: >=1 commentary frame (scenario 5) --assert-read-receipt M7: a read.receipt frame arrives after the sent message. SKIPs (exit 0) when the frame never arrives (old gateway without the M7 frame). --assert-status M7: a status frame is received. SKIPs (exit 0) when the frame never arrives. Request modes (no turn driven unless --send/--upload also given): --search Q [--scope all|chat] [--chat-id C] M3: send search {query, scope, limit} and assert >=1 hit in search.results (scenario 8). With --send, the turn is driven first, then the search runs. --channel-create NAME M3: send channel.create, print the new chat_id ("== channel created: "), exit --channel-delete CHAT M3: send channel.delete, assert channel.deleted --channel-list M3: send channel.list, print the directory --watch CHAT_ID wait up to --timeout for a message to land in CHAT_ID (cron delivery E2E, scenario 7) HTTP fallback leg (docs/19): --http drive the turn over the HTTP leg instead of WS: GET /v1/health, POST /v1/frame (message.send), receive over SSE /v1/events. The same assertion flags apply. --http-url http://host:port base for --http (default: derived from --url, ws(s) -> http(s), port 8791) --http-media FILE with --http: also upload FILE via POST /v1/media (docs/19 §19.15, v2) and assert a 201 ack Exit codes: 0 ok (incl. SKIP for absent M7 frames) 2 connect failed 3 no hello.ack 4 expected hello.ack, got something else 5 --authfail but the token was accepted 6 timeout waiting for the final message 7 no final assistant message 8 upload/sync failed 9 media pull failed 10 --assert-turn failed (no ordered start/update/stop segment) 11 --assert-reasoning failed (final has no non-empty reasoning) 12 --assert-tools failed (no tool.start with a matching tool.end) 13 --assert-commentary failed (no commentary frame) 14 --search failed (error or zero hits) 15 --channel-create / --channel-list failed 16 --channel-delete failed 17 --watch timed out (no message landed in the channel) 18 --assert-read-receipt failed (frame arrived before the sent message) 19 --assert-status failed (status frame arrived with an empty payload) 20 --http: health check failed 21 --http: SSE open failed 22 --http: POST /v1/frame rejected (4xx) 23 --http: POST /v1/media rejected (media upload, v2) """ import argparse import hashlib import json import mimetypes import os import sys import time import uuid def _print_frame(raw): try: data = json.loads(raw) except (json.JSONDecodeError, TypeError): print(f" <- {raw!r}") return None ftype = data.get("type", "?") chat = data.get("chat_id") fid = data.get("id") payload = data.get("payload", {}) # Compact one-line summary + full payload for the interesting frames. extra = "" if ftype == "message": text = payload.get("text") or "" extra = f" role={payload.get('role')} id={payload.get('message_id')} text={text[:120]!r}" if payload.get("reasoning"): extra += f" reasoning={payload['reasoning'][:80]!r}" elif ftype == "message.start": extra = f" id={payload.get('message_id')} role={payload.get('role')}" elif ftype == "message.update": text = payload.get("text") or "" extra = f" id={payload.get('message_id')} text={text[:100]!r}" elif ftype == "message.stop": text = payload.get("final_text") or "" extra = f" id={payload.get('message_id')} text={text[:120]!r}" if payload.get("reasoning"): extra += f" reasoning={payload['reasoning'][:80]!r}" elif ftype == "tool.start": extra = ( f" idx={payload.get('index')} name={payload.get('name')!r} " f"preview={str(payload.get('preview'))[:80]!r}" ) elif ftype == "tool.progress": extra = ( f" idx={payload.get('index')} name={payload.get('name')!r} note={payload.get('note')!r}" ) elif ftype == "tool.end": extra = ( f" idx={payload.get('index')} name={payload.get('name')!r} " f"ok={payload.get('ok')} dur={payload.get('duration')}" ) elif ftype == "commentary": extra = f" id={payload.get('message_id')} text={(payload.get('text') or '')[:120]!r}" elif ftype == "hello.ack": extra = f" caps={payload.get('server_caps')} cursor={payload.get('sync_cursor')}" elif ftype == "error": extra = f" code={payload.get('code')} msg={payload.get('message')!r}" elif ftype == "typing": extra = f" on={payload.get('on')}" elif ftype == "pong": extra = "" elif ftype == "media.offer": extra = ( f" media_id={payload.get('media_id')} kind={payload.get('kind')} " f"mime={payload.get('mime')} size={payload.get('size')} " f"file={payload.get('filename')!r} msg={payload.get('message_id')}" ) elif ftype == "media.upload.ack": extra = f" ok={payload.get('ok')} ref={payload.get('media_ref')}" elif ftype == "media.pull.end": extra = f" ok={payload.get('ok')}" elif ftype == "notification": extra = ( f" kind={payload.get('kind')} title={payload.get('title')!r} " f"body={(payload.get('body') or '')[:100]!r}" ) elif ftype in {"sync", "sync.done"}: extra = f" cursor={payload.get('cursor')}" elif ftype == "search.results": hits = payload.get("hits") or [] extra = f" query={payload.get('query')!r} scope={payload.get('scope')} hits={len(hits)}" elif ftype == "channel.created": extra = f" chat_id={payload.get('chat_id')} name={payload.get('name')!r}" elif ftype == "channel.deleted": extra = f" chat_id={payload.get('chat_id')}" elif ftype == "channel.list": extra = f" channels={len(payload.get('channels') or [])}" elif ftype in {"read.receipt", "status"}: extra = f" payload={ {k: payload[k] for k in list(payload)[:4]} }" scope = f" chat={chat}" if chat else "" idpart = f" id={fid}" if fid is not None else "" print(f" <- {ftype}{idpart}{scope}{extra}") return data def _kind_for_path(path: str) -> str: mime, _ = mimetypes.guess_type(path) mime = mime or "application/octet-stream" if mime.startswith("image/"): return "image" if mime.startswith("video/"): return "video" if mime.startswith("audio/"): return "audio" return "document" class _TurnState: """Assertion-relevant facts collected while driving a turn.""" def __init__(self): self.seq: list[tuple[str, str | None]] = [] # (type, message_id) self.tool_starts: set[int] = set() self.tool_ends: set[int] = set() self.commentary = 0 self.final_stop_reasoning: str | None = None self.final_message_reasoning: str | None = None self.user_echo_seen = False self.read_receipt: bool | None = None # None = never arrived self.status_seen = False self.status_empty = False self.pulled = False def track(self, ftype: str, payload: dict) -> None: if ftype in ("message.start", "message.update", "message.stop"): self.seq.append((ftype, payload.get("message_id"))) if ftype == "message.stop": r = payload.get("reasoning") if isinstance(r, str) and r.strip(): self.final_stop_reasoning = r if ftype == "message" and payload.get("role") == "assistant": r = payload.get("reasoning") if isinstance(r, str) and r.strip(): self.final_message_reasoning = r if ftype == "message" and payload.get("role") == "user": self.user_echo_seen = True if ftype == "tool.start" and isinstance(payload.get("index"), int): self.tool_starts.add(payload["index"]) if ftype == "tool.end" and isinstance(payload.get("index"), int): self.tool_ends.add(payload["index"]) if ftype == "commentary": self.commentary += 1 if ftype == "read.receipt": self.read_receipt = self.user_echo_seen if ftype == "status": self.status_seen = True if not payload: self.status_empty = True def _evaluate_assertions(args, st: _TurnState) -> list[tuple[int, bool, str]]: """Evaluate the enabled assertion modes. Returns (exit_code, ok, message) per failed-or-passed assertion; SKIPs are printed here and not returned.""" results: list[tuple[int, bool, str]] = [] if args.assert_turn: ok = False for mid in {m for _, m in st.seq if m is not None}: events = [t for t, m in st.seq if m == mid] if "message.start" in events and "message.stop" in events: i_start = events.index("message.start") i_stop = events.index("message.stop") if any(i_start < i < i_stop for i, e in enumerate(events) if e == "message.update"): ok = True break results.append( (10, ok, "assert-turn: no message.start -> >=1 message.update -> message.stop") ) if args.assert_reasoning: reasoning = st.final_stop_reasoning or st.final_message_reasoning results.append( (11, bool(reasoning), "assert-reasoning: final message has no non-empty reasoning") ) if args.assert_tools: ok = bool(st.tool_starts) and bool(st.tool_starts & st.tool_ends) results.append((12, ok, "assert-tools: no tool.start with a matching tool.end")) if args.assert_commentary: results.append((13, st.commentary >= 1, "assert-commentary: no commentary frame")) if args.assert_read_receipt: if st.read_receipt is None: print("== SKIP: no read.receipt frame (M7 frame not live on this gateway)") elif not st.read_receipt: results.append( (18, False, "assert-read-receipt: read.receipt arrived before the sent message") ) if args.assert_status: if not st.status_seen: print("== SKIP: no status frame (M7 frame not live on this gateway)") elif st.status_empty: results.append((19, False, "assert-status: status frame arrived with an empty payload")) return results def _http_frame_roundtrip( host: str, port: int, headers: dict, frame: dict, expect_type: str, timeout: float ) -> dict | None: """Open SSE, POST *frame*, and read the first SSE frame of *expect_type*. Returns the parsed response frame, or None on timeout. Used by the request/response probe modes (channel ops, search, sync, fcm.register). """ from http.client import HTTPConnection sse = HTTPConnection(host, port, timeout=timeout) sse.request("GET", "/v1/events", headers=headers) resp = sse.getresponse() if resp.status != 200: print(f"!! SSE open failed: HTTP {resp.status}") sse.close() return None conn = HTTPConnection(host, port, timeout=30) conn.request( "POST", "/v1/frame", body=json.dumps(frame), headers={**headers, "Content-Type": "application/json"}, ) r = conn.getresponse() body = r.read() conn.close() print(f"== POST /v1/frame ({frame.get('type')}) -> {r.status} {body[:160]!r}") if r.status >= 400: return None deadline = time.time() + timeout cur: list[str] = [] sock = getattr(getattr(resp.fp, "raw", None), "_sock", None) try: while time.time() < deadline: if sock is not None: sock.settimeout(max(0.1, deadline - time.time())) line = resp.fp.readline() if not line: break line = line.decode("utf-8").rstrip("\r\n") if line == "": if cur: data = _print_frame("\n".join(cur)) cur = [] if data is not None and data.get("type") == expect_type: return data elif not line.startswith(":"): field, _, value = line.partition(":") if value.startswith(" "): value = value[1:] if field == "data": cur.append(value) finally: sse.close() return None def run_http(args, base: str) -> int: """docs/19: drive a turn over the HTTP fallback leg — GET /v1/health, POST /v1/frame (message.send), receive over SSE /v1/events. Blocking (stdlib http.client); the same assertion flags apply as the WS leg.""" from http.client import HTTPConnection from urllib.parse import urlparse u = urlparse(base) host = u.hostname or "127.0.0.1" port = u.port or (443 if u.scheme == "https" else 80) headers = { "Authorization": f"Bearer {args.token}", "X-Iris-Device": args.device, } # 1. health (unauthenticated liveness probe). try: conn = HTTPConnection(host, port, timeout=5) conn.request("GET", "/v1/health") r = conn.getresponse() body = r.read() conn.close() except Exception as e: print(f"!! health check failed: {e}") return 20 if r.status != 200: print(f"!! health check failed: HTTP {r.status} {body[:200]!r}") return 20 print(f"== health ok: {body!r}") # Request/response probe modes (channel ops, search, sync, fcm.register, # watch, authfail): a single frame round-trip over SSE, then exit. if args.authfail: sse = HTTPConnection(host, port, timeout=10) sse.request("GET", "/v1/events", headers=headers) rr = sse.getresponse() rr.read() sse.close() if rr.status == 401: print("== auth rejected as expected (401)") return 0 print(f"!! expected 401, got {rr.status}") return 30 if args.channel_create: d = _http_frame_roundtrip( host, port, headers, {"v": 1, "id": 1, "type": "channel.create", "payload": {"name": args.channel_create}}, "channel.created", 30, ) if d is None: print("!! channel.create: no channel.created") return 15 print(f"== channel created: {(d.get('payload') or {}).get('chat_id')}") return 0 if args.channel_delete: d = _http_frame_roundtrip( host, port, headers, { "v": 1, "id": 1, "type": "channel.delete", "payload": {"chat_id": args.channel_delete}, }, "channel.deleted", 30, ) if d is None: print("!! channel.delete: no channel.deleted") return 16 print(f"== channel deleted: {args.channel_delete}") return 0 if args.channel_list: d = _http_frame_roundtrip( host, port, headers, {"v": 1, "id": 1, "type": "channel.list", "payload": {}}, "channel.list", 30, ) if d is None: print("!! channel.list: no response") return 15 for c in (d.get("payload") or {}).get("channels") or []: print( f"== channel: {c.get('chat_id')} name={c.get('name')!r} default={bool(c.get('is_default'))}" ) return 0 if args.search: payload = {"query": args.search, "scope": args.scope, "limit": 20} if args.scope == "chat": payload["chat_id"] = args.chat_id d = _http_frame_roundtrip( host, port, headers, {"v": 1, "id": 1, "type": "search", "payload": payload}, "search.results", 30, ) if d is None: print("!! search: no search.results") return 14 hits = (d.get("payload") or {}).get("hits") or [] print(f"== search: {len(hits)} hit(s)") for h in hits[:10]: print( f" hit chat={h.get('chat_id')} role={h.get('role')} snippet={str(h.get('snippet'))[:100]!r}" ) return 0 if hits else 14 if args.sync is not None: d = _http_frame_roundtrip( host, port, headers, {"v": 1, "id": 1, "type": "sync", "payload": {"cursor": args.sync}}, "sync.done", 30, ) if d is None: print("!! sync: no sync.done") return 18 print(f"== sync done: cursor={(d.get('payload') or {}).get('cursor')}") return 0 if args.fcm_reg: d = _http_frame_roundtrip( host, port, headers, {"v": 1, "id": 1, "type": "fcm.register", "payload": {"token": args.fcm_token}}, "fcm.registered", 30, ) if d is None: print("!! fcm.register: no fcm.registered") return 19 print("== fcm registered") return 0 if args.watch: print(f"== watching {args.watch} for a message (timeout {args.timeout:.0f}s)") sse = HTTPConnection(host, port, timeout=args.timeout) sse.request("GET", "/v1/events", headers=headers) resp = sse.getresponse() if resp.status != 200: print(f"!! SSE open failed: HTTP {resp.status}") sse.close() return 17 deadline = time.time() + args.timeout cur: list[str] = [] sock = getattr(getattr(resp.fp, "raw", None), "_sock", None) try: while time.time() < deadline: if sock is not None: sock.settimeout(max(0.1, deadline - time.time())) line = resp.fp.readline() if not line: break line = line.decode("utf-8").rstrip("\r\n") if line == "": if cur: data = _print_frame("\n".join(cur)) cur = [] if data is not None and data.get("chat_id") == args.watch: ftype = data.get("type") payload = data.get("payload") or {} if ftype == "message" and payload.get("role") in ("assistant", "cron"): print( f"== message landed in {args.watch}: {str(payload.get('text'))[:120]!r}" ) return 0 elif not line.startswith(":"): field, _, value = line.partition(":") if value.startswith(" "): value = value[1:] if field == "data": cur.append(value) finally: sse.close() print(f"!! no message landed in {args.watch}") return 17 # 1b. optional media upload round-trip (docs/19 §19.15, v2). if args.http_media: import mimetypes with open(args.http_media, "rb") as f: data = f.read() mime, _ = mimetypes.guess_type(args.http_media) kind = "image" if (mime or "").startswith("image/") else "document" conn = HTTPConnection(host, port, timeout=60) conn.request( "POST", "/v1/media", body=data, headers={ **headers, "Content-Type": mime or "application/octet-stream", "X-Iris-Media-Ref": f"probe_{uuid.uuid4().hex[:12]}", "X-Iris-Media-Kind": kind, "X-Iris-Media-Filename": os.path.basename(args.http_media), "X-Iris-Media-Sha256": hashlib.sha256(data).hexdigest(), }, ) r = conn.getresponse() body = r.read() conn.close() print(f"== POST /v1/media ({len(data)} bytes) -> {r.status} {body[:200]!r}") if r.status != 201: print("!! media upload rejected") return 23 # 2. open the SSE stream. sse = HTTPConnection(host, port, timeout=args.timeout) sse.request("GET", "/v1/events", headers=headers) resp = sse.getresponse() if resp.status != 200: print(f"!! SSE open failed: HTTP {resp.status}") return 21 print("== SSE open (/v1/events)") # 3. POST the message.send frame (accept-and-ack). post_time: float | None = None if args.send: frame = { "v": 1, "id": 1, "type": "message.send", "chat_id": "default", "payload": {"text": args.send}, } conn = HTTPConnection(host, port, timeout=30) conn.request( "POST", "/v1/frame", body=json.dumps(frame), headers={**headers, "Content-Type": "application/json"}, ) r = conn.getresponse() body = r.read() conn.close() post_time = time.time() print(f"== POST /v1/frame -> {r.status} {body[:200]!r}") if r.status >= 400: print("!! POST /v1/frame rejected") return 22 # 4. read SSE until the final assistant message (same final-detection # logic as the WS leg). st = _TurnState() got_final = False seen_final_frame = False echo_logged = False deadline = time.time() + args.timeout cur_data: list[str] = [] def feed(line: str) -> bool: nonlocal cur_data, got_final, seen_final_frame, echo_logged line = line.rstrip("\r\n") if line == "": if cur_data: data = _print_frame("\n".join(cur_data)) if data is not None: ftype = data.get("type") payload = data.get("payload") or {} st.track(ftype, payload) # docs/19: the user echo must land on the SSE stream # promptly after the POST (the < 1 s sendable-in-fallback # UX assertion). if ( not echo_logged and post_time is not None and ftype == "message" and payload.get("role") == "user" ): echo_logged = True print(f"== user echo in {time.time() - post_time:.2f}s") if ftype == "message" and payload.get("role") == "assistant": got_final = True if ftype == "message.stop": seen_final_frame = True if ftype == "typing" and payload.get("on") is False and seen_final_frame: got_final = True cur_data = [] return got_final if line.startswith(":"): return got_final # heartbeat comment field, _, value = line.partition(":") if value.startswith(" "): value = value[1:] if field == "data": cur_data.append(value) return got_final sock = getattr(getattr(resp.fp, "raw", None), "_sock", None) try: while time.time() < deadline and not got_final: if sock is not None: sock.settimeout(max(0.1, deadline - time.time())) line = resp.fp.readline() if not line: break if feed(line.decode("utf-8")): break finally: sse.close() if not got_final: print(f"!! no final assistant message (HTTP leg, {args.timeout:.0f}s)") return 7 print("== final assistant message received (via SSE)") for code, ok, msg in _evaluate_assertions(args, st): if not ok: print(f"!! {msg}") return code return 0 def main() -> int: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--url", default=os.getenv("IRIS_WS_URL", "ws://127.0.0.1:8790/ws")) p.add_argument("--token", default=os.getenv("IRIS_TOKEN", "")) p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}") p.add_argument("--send", default="hello") p.add_argument( "--upload", default="", help="M4: file to upload (chunked) and attach via media_refs" ) p.add_argument( "--pull-offer", action="store_true", help="M4: pull any media.offer that arrives during the turn", ) p.add_argument( "--sync", type=int, default=None, help="M5: send sync {cursor} after pairing, print replay, exit", ) p.add_argument("--fcm-token", default="", help="M5: FCM token to attach to the hello payload") p.add_argument( "--fcm-reg", action="store_true", help="M5: send fcm.register after pairing (uses --fcm-token)", ) p.add_argument("--timeout", type=float, default=120.0) p.add_argument("--authfail", action="store_true", help="expect an auth rejection (wrong token)") p.add_argument( "--assert-turn", action="store_true", help="assert message.start -> >=1 message.update -> message.stop", ) p.add_argument( "--assert-reasoning", action="store_true", help="assert the final message.stop carries non-empty reasoning", ) p.add_argument( "--assert-tools", action="store_true", help="assert >=1 tool.start with a matching tool.end" ) p.add_argument("--assert-commentary", action="store_true", help="assert >=1 commentary frame") p.add_argument( "--assert-read-receipt", action="store_true", help="assert a read.receipt arrives after the sent message (SKIP if absent; M7)", ) p.add_argument( "--assert-status", action="store_true", help="assert a status frame is received (SKIP if absent; M7)", ) p.add_argument( "--search", default="", help="M3: send search {query, scope, limit}, assert >=1 hit" ) p.add_argument( "--scope", choices=("all", "chat"), default="all", help="search scope (default all)" ) p.add_argument( "--chat-id", default="default", help="chat_id for --scope chat (default default)", ) p.add_argument( "--channel-create", default="", help="M3: create a channel, print its chat_id, exit" ) p.add_argument("--channel-delete", default="", help="M3: delete (archive) a channel, exit") p.add_argument("--channel-list", action="store_true", help="M3: list channels, exit") p.add_argument( "--watch", default="", help="wait up to --timeout for a message to land in this chat_id" ) p.add_argument( "--offer-grace", type=float, default=15.0, help="seconds to wait for a media.offer after the final " "message when --pull-offer (default 15)", ) p.add_argument( "--http", action="store_true", help="docs/19: drive the turn over the HTTP fallback leg " "(health + POST /v1/frame + SSE /v1/events) instead of WS", ) p.add_argument( "--http-url", default="", help="docs/19: http(s)://host:port base for --http " "(default: derived from --url, port 8791)", ) p.add_argument( "--http-media", default="", help="docs/19 v2: with --http, also upload this file via POST /v1/media " "and assert a 201 ack", ) args = p.parse_args() if not args.token and not args.authfail: p.error("--token (or $IRIS_TOKEN) is required") if args.assert_read_receipt and not args.send: p.error("--assert-read-receipt requires --send (the receipt must follow the sent message)") # HTTP is the only transport (docs/19): derive the http(s) base from the # --url (ws://host:8790/ws -> http://host:8791) unless --http-url is given. if args.http_url: base = args.http_url else: from urllib.parse import urlparse u = urlparse(args.url) scheme = "https" if u.scheme == "wss" else "http" base = f"{scheme}://{u.hostname or '127.0.0.1'}:8791" return run_http(args, base) if __name__ == "__main__": sys.exit(main())