#!/usr/bin/env python3 """E2E driver: docs/13-testing.md §13.4 scenarios 1-12 against the live gateway. Drives ws_probe.py (and the hermes CLI for cron) as subprocesses. For each scenario prints PASS / PARTIAL / SKIP / FAIL with a one-line reason, then a summary table. Exit 0 if no FAIL, 1 otherwise. Usage:: hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --skip 3,5,7 hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --url ws://host:8790/ws The token is read from $IRIS_TOKEN, else hermes-agent/.env, else ~/.hermes/.env. The gateway must already be running (this driver never starts or stops it). Idempotent: channels/jobs it creates are cleaned up even on failure, and leftover "e2e-*" channels/jobs from earlier runs are removed at start. Scenario notes: 3 (reasoning) and 5 (commentary) are model-dependent: they SKIP (not FAIL) when the current model does not emit reasoning / commentary. 11 (push) and 12 (reconnect) are PARTIAL by design: the WS leg is automated, the device-notification / gateway-kill leg is manual. """ import argparse import os import re import struct import subprocess import sys import uuid import zlib from pathlib import Path from urllib.parse import urlparse HERE = Path(__file__).resolve().parent REPO = HERE.parent.parent PY = REPO / "hermes-agent" / ".venv" / "bin" / "python" PROBE = HERE / "ws_probe.py" HERMES = REPO / "hermes-agent" / ".venv" / "bin" / "hermes" DEFAULT_URL = "ws://127.0.0.1:8790/ws" PASS, PARTIAL, SKIP, FAIL = "PASS", "PARTIAL", "SKIP", "FAIL" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def find_token(cli_token: str) -> str: if cli_token: return cli_token env = os.getenv("IRIS_TOKEN") if env: return env for p in (REPO / "hermes-agent" / ".env", Path.home() / ".hermes" / ".env"): try: for raw_line in p.read_text().splitlines(): line = raw_line.strip() if line.startswith("IRIS_TOKEN="): return line.split("=", 1)[1].strip().strip('"').strip("'") except OSError: pass return "" def run_probe(env, url, token, *args, timeout=300): cmd = [str(PY), str(PROBE), "--url", url, "--token", token, *args] p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env) return p.returncode, p.stdout, p.stderr def run_hermes(env, *args, timeout=120): cmd = [str(HERMES), *args] p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env) return p.returncode, p.stdout, p.stderr def write_png(path: Path, color, size: int = 200) -> None: """Write a solid-color RGB PNG using only the stdlib (no PIL needed).""" raw = b"".join(b"\x00" + bytes(color) * size for _ in range(size)) def chunk(tag: bytes, data: bytes) -> bytes: return (struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)) ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0) path.write_bytes( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") ) def parse_created_chat_id(out: str) -> str | None: m = re.search(r"== channel created: (\S+)", out) return m.group(1) if m else None def sweep_leftovers(env, url, token) -> None: """Remove e2e-* channels / cron jobs left behind by earlier runs.""" rc, out, _ = run_probe(env, url, token, "--channel-list") if rc == 0: for m in re.finditer(r"== channel: (\S+) name='(e2e-[^']*)'", out): chat_id, name = m.group(1), m.group(2) print(f" cleanup: removing leftover channel {chat_id} ({name})") run_probe(env, url, token, "--channel-delete", chat_id) rc, out, _ = run_hermes(env, "cron", "list") if rc == 0: for m in re.finditer( r"(\S+) \[(?:active|paused)\]\s*\n\s*Name:\s+(e2e-cron-[^ \n]*)", out ): job_id, name = m.group(1), m.group(2) print(f" cleanup: removing leftover cron job {job_id} ({name})") run_hermes(env, "cron", "remove", job_id) # --------------------------------------------------------------------------- # Scenarios (docs/13-testing.md §13.4) # --------------------------------------------------------------------------- def s1_pair(env, url, token): rc, _, _ = run_probe(env, url, "definitely-wrong-token", "--authfail", "--send", "") if rc != 0: return FAIL, f"wrong token was not rejected (rc={rc})" rc, _, _ = run_probe(env, url, token, "--send", "") if rc != 0: return FAIL, f"valid token did not pair (rc={rc})" return PASS, "wrong token rejected; hello.ack on valid token" def s2_text(env, url, token): prompt = "Write a short poem about the ocean, at least 8 lines" rc, _, _ = run_probe(env, url, token, "--send", prompt, "--assert-turn", "--timeout", "120") if rc == 0: return PASS, "message.start -> >=1 message.update -> message.stop" if rc == 10: return FAIL, "no ordered start/update/stop segment" return FAIL, f"probe rc={rc}" def s3_reasoning(env, url, token): prompt = "Work out step by step: what is 17 * 23? Show your reasoning." rc, _, _ = run_probe(env, url, token, "--send", prompt, "--assert-reasoning", "--timeout", "120") if rc == 0: return PASS, "final message.stop carries non-empty reasoning" if rc == 11: return SKIP, "model returned no reasoning (model-dependent)" return FAIL, f"probe rc={rc}" def s4_tools(env, url, token): prompt = ("List the files in your current working directory using your " "shell tool, then tell me how many there are") rc, _, _ = run_probe(env, url, token, "--send", prompt, "--assert-tools", "--timeout", "150") if rc == 0: return PASS, "tool.start with a matching tool.end" if rc == 12: return FAIL, "no tool.start/tool.end pair" return FAIL, f"probe rc={rc}" def s5_commentary(env, url, token): prompt = ("Research task: (1) use your shell tool to list the top-level " "directories in /tmp, (2) report your findings so far, " "(3) use your shell tool to count files in /tmp, " "(4) report those findings too, (5) give a final summary of both") rc, _, _ = run_probe(env, url, token, "--send", prompt, "--assert-commentary", "--timeout", "150") if rc == 0: return PASS, "commentary frame observed" if rc == 13: return SKIP, "no commentary (model/agent-dependent per M2)" return FAIL, f"probe rc={rc}" def s6_channels(env, url, token): name = f"e2e-chan-{uuid.uuid4().hex[:6]}" rc, out, _ = run_probe(env, url, token, "--channel-create", name) if rc != 0: return FAIL, f"channel.create failed (rc={rc})" chat_id = parse_created_chat_id(out) if not chat_id: return FAIL, "channel.created received but chat_id not parseable" rc, _, _ = run_probe(env, url, token, "--channel-delete", chat_id) if rc != 0: run_probe(env, url, token, "--channel-delete", chat_id) # best-effort return FAIL, f"channel.delete failed (rc={rc})" return PASS, f"created {chat_id} + deleted (cleanup)" def s7_cron(env, url, token): chan_name = f"e2e-cron-chan-{uuid.uuid4().hex[:6]}" rc, out, _ = run_probe(env, url, token, "--channel-create", chan_name) if rc != 0: return SKIP, f"could not create cron target channel (rc={rc})" chat_id = parse_created_chat_id(out) if not chat_id: return FAIL, "channel.created received but chat_id not parseable" job_name = f"e2e-cron-{uuid.uuid4().hex[:6]}" deliver = f"iris:{chat_id}" rc, out, err = run_hermes( env, "cron", "create", "1m", "Reply with exactly: e2e cron delivery OK", "--deliver", deliver, "--name", job_name, ) job_id = None if rc == 0: m = re.search(r"Created job: (\S+)", out) job_id = m.group(1) if m else None try: if rc != 0: return SKIP, f"hermes cron create failed: {(err or out).strip()[:120]}" rc, out, _ = run_probe(env, url, token, "--watch", chat_id, "--timeout", "330", timeout=400) if rc == 0: return PASS, f"one-shot cron job fired; message landed in {chat_id}" return FAIL, f"no message in {chat_id} within 330s (probe rc={rc})" finally: if job_id: run_hermes(env, "cron", "remove", job_id) else: # create succeeded but the id was not parseable: find by name. _, list_out, _ = run_hermes(env, "cron", "list") m = re.search(r"(\S+) \[active\]\s*\n\s*Name:\s+" + re.escape(job_name), list_out) if m: run_hermes(env, "cron", "remove", m.group(1)) run_probe(env, url, token, "--channel-delete", chat_id) def s8_search(env, url, token): marker = f"e2emarker{uuid.uuid4().hex[:8]}" rc, _, _ = run_probe(env, url, token, "--send", f"Remember this marker phrase: {marker}. " "Just acknowledge it briefly.", "--timeout", "120") if rc != 0: return FAIL, f"setup message failed (rc={rc})" rc, _, _ = run_probe(env, url, token, "--send", "", "--search", marker) if rc == 0: return PASS, f"search for {marker!r} returned >=1 hit" if rc == 14: return FAIL, f"search for {marker!r} returned 0 hits" return FAIL, f"probe rc={rc}" def s9_media_in(env, url, token): png = Path(f"/tmp/e2e_in_{uuid.uuid4().hex[:6]}.png") write_png(png, (30, 120, 220)) try: rc, _, _ = run_probe(env, url, token, "--upload", str(png), "--send", "describe this image briefly", "--timeout", "120") if rc == 0: return PASS, "upload + vision reply (final message)" if rc == 8: return FAIL, "media upload failed" if rc == 7: return FAIL, "no final message after upload" return FAIL, f"probe rc={rc}" finally: png.unlink(missing_ok=True) def s10_media_out(env, url, token): prompt = ("Create a 100x100 orange square PNG in /tmp with your tools. " "In your final reply, include the MEDIA:/absolute/path tag for " "that file so it is delivered to me.") rc, out, _ = run_probe(env, url, token, "--send", prompt, "--pull-offer", "--timeout", "150") m = re.search(r"== pulled (\d+) bytes", out) if rc == 0 and m and int(m.group(1)) > 0: return PASS, f"media.offer pulled ({m.group(1)} bytes)" if rc == 9: return FAIL, "media pull failed" return FAIL, "no media.offer pulled (agent did not deliver an image)" def s11_push(env, url, token): rc, out, _ = run_probe(env, url, token, "--fcm-token", "test-token-123", "--fcm-reg", "--send", "") if rc != 0: return FAIL, f"probe rc={rc}" if "<- error" in out: return FAIL, "error frame after fcm.register" return PARTIAL, ("fcm.register accepted (no error frame); " "device-notification leg is manual") def s12_sync(env, url, token): rc, out, _ = run_probe(env, url, token, "--sync", "0") if rc == 0 and "sync done" in out: return PARTIAL, ("sync replay + sync.done verified; " "gateway-kill/restart leg is manual") if rc == 8: return FAIL, "sync failed" return FAIL, f"probe rc={rc}" def s13_http_fallback(env, url, token): """docs/19: the HTTP fallback leg. The probe drives a full turn over health + POST /v1/frame + SSE /v1/events (no WS involved). The user echo must land on the SSE stream promptly after the POST (< 1.5 s on LAN).""" u = urlparse(url) scheme = "https" if u.scheme == "wss" else "http" http_port = os.getenv("IRIS_HTTP_PORT", "8791") http_url = f"{scheme}://{u.hostname or '127.0.0.1'}:{http_port}" rc, out, _ = run_probe(env, url, token, "--http", "--http-url", http_url, "--send", "Reply with exactly: e2e http fallback OK", "--timeout", "120") if rc == 0: m = re.search(r"== user echo in ([\d.]+)s", out) echo = float(m.group(1)) if m else None if echo is not None and echo > 1.5: return FAIL, f"user echo took {echo:.2f}s (> 1.5 s)" return PASS, ("health + POST /v1/frame + SSE turn complete" + (f"; user echo in {echo:.2f}s" if echo is not None else "")) if rc == 20: return FAIL, "health check failed (HTTP leg not running?)" if rc == 21: return FAIL, "SSE open failed" if rc == 22: return FAIL, "POST /v1/frame rejected" return FAIL, f"probe rc={rc}" SCENARIOS = [ (1, "pair", s1_pair), (2, "text round-trip", s2_text), (3, "reasoning", s3_reasoning), (4, "tools", s4_tools), (5, "commentary", s5_commentary), (6, "channels", s6_channels), (7, "cron delivery", s7_cron), (8, "search", s8_search), (9, "media in", s9_media_in), (10, "media out", s10_media_out), (11, "push", s11_push), (12, "reconnect/sync", s12_sync), (13, "http fallback", s13_http_fallback), ] def main() -> int: p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) p.add_argument("--url", default=os.getenv("IRIS_WS_URL", DEFAULT_URL)) p.add_argument("--token", default="") p.add_argument("--skip", default="", help="comma-separated scenario numbers to skip (e.g. 3,5,7)") args = p.parse_args() token = find_token(args.token) if not token: print("!! IRIS_TOKEN not found (env, hermes-agent/.env, or ~/.hermes/.env)") return 1 skip = {int(x) for x in args.skip.split(",") if x.strip()} env = dict(os.environ) env["IRIS_TOKEN"] = token print(f"== e2e: url={args.url} token={token[:6]}…") sweep_leftovers(env, args.url, token) results = [] for num, name, fn in SCENARIOS: if num in skip: results.append((num, name, SKIP, "skipped by --skip")) print(f"[{num:2d}] {name:<18} {SKIP:<7} skipped by --skip") continue print(f"[{num:2d}] {name:<18} running…", flush=True) try: status, reason = fn(env, args.url, token) except Exception as e: status, reason = FAIL, f"driver error: {e}" results.append((num, name, status, reason)) print(f"[{num:2d}] {name:<18} {status:<7} {reason}") print() print("=" * 78) print(f"{'#':<3} {'scenario':<18} {'status':<8} reason") print("-" * 78) for num, name, status, reason in results: print(f"{num:<3} {name:<18} {status:<8} {reason}") print("-" * 78) counts = {s: sum(1 for r in results if r[2] == s) for s in (PASS, PARTIAL, SKIP, FAIL)} print(f"total: {len(results)} PASS={counts[PASS]} PARTIAL={counts[PARTIAL]} " f"SKIP={counts[SKIP]} FAIL={counts[FAIL]}") return 1 if counts[FAIL] else 0 if __name__ == "__main__": sys.exit(main())