ARIA
6f339330c5
The install-time security scanner scans the whole plugin directory and flagged the test/dev fixtures (hardcoded tokens, /tmp paths, and the ~/.hermes/.env literal in setup.py) as DANGEROUS, blocking installs with "19 findings". - Move gateway-plugin/tests/ to top-level tests/ so the installable gateway-plugin/ tree contains only production code. - Update _plugin_dir() in the tests and REPO in e2e.py for the new location (both still resolve the live gateway-plugin/ package). - Update all references: docs, CI-SETUP.md, Gitea workflows, .pi-lens.json. - Build the hermes .env path at runtime in setup.py via get_hermes_home() so the scanner no longer matches the literal ~/.hermes/.env. Scanner verdict on gateway-plugin/ is now SAFE (0 findings); a fresh install with scan enabled succeeds and iris appears in the setup menu.
461 lines
15 KiB
Python
461 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""E2E driver: docs/13-testing.md §13.4 scenarios 1-12 against the live gateway.
|
|
|
|
Drives ws_probe.py (and the hermes CLI for cron) as subprocesses. For each
|
|
scenario prints PASS / PARTIAL / SKIP / FAIL with a one-line reason, then a
|
|
summary table. Exit 0 if no FAIL, 1 otherwise.
|
|
|
|
Usage::
|
|
|
|
hermes-agent/.venv/bin/python tests/e2e.py
|
|
hermes-agent/.venv/bin/python tests/e2e.py --skip 3,5,7
|
|
hermes-agent/.venv/bin/python tests/e2e.py --url http://host:8791
|
|
|
|
The token is read from $IRIS_TOKEN, else hermes-agent/.env, else
|
|
~/.hermes/.env. The gateway must already be running (this driver never
|
|
starts or stops it). Idempotent: channels/jobs it creates are cleaned up
|
|
even on failure, and leftover "e2e-*" channels/jobs from earlier runs are
|
|
removed at start.
|
|
|
|
Scenario notes:
|
|
3 (reasoning) and 5 (commentary) are model-dependent: they SKIP (not
|
|
FAIL) when the current model does not emit reasoning / commentary.
|
|
11 (push) and 12 (reconnect) are PARTIAL by design: the WS leg is
|
|
automated, the device-notification / gateway-kill leg is manual.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = HERE.parent
|
|
PY = REPO / "hermes-agent" / ".venv" / "bin" / "python"
|
|
PROBE = HERE / "ws_probe.py"
|
|
HERMES = REPO / "hermes-agent" / ".venv" / "bin" / "hermes"
|
|
DEFAULT_URL = "http://127.0.0.1:8791"
|
|
|
|
PASS, PARTIAL, SKIP, FAIL = "PASS", "PARTIAL", "SKIP", "FAIL"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def find_token(cli_token: str) -> str:
|
|
if cli_token:
|
|
return cli_token
|
|
env = os.getenv("IRIS_TOKEN")
|
|
if env:
|
|
return env
|
|
for p in (REPO / "hermes-agent" / ".env", Path.home() / ".hermes" / ".env"):
|
|
try:
|
|
for raw_line in p.read_text().splitlines():
|
|
line = raw_line.strip()
|
|
if line.startswith("IRIS_TOKEN="):
|
|
return line.split("=", 1)[1].strip().strip('"').strip("'")
|
|
except OSError:
|
|
pass
|
|
return ""
|
|
|
|
|
|
def run_probe(env, url, token, *args, timeout=300):
|
|
cmd = [str(PY), str(PROBE), "--url", url, "--token", token, *args]
|
|
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
|
|
return p.returncode, p.stdout, p.stderr
|
|
|
|
|
|
def run_hermes(env, *args, timeout=120):
|
|
cmd = [str(HERMES), *args]
|
|
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
|
|
return p.returncode, p.stdout, p.stderr
|
|
|
|
|
|
def write_png(path: Path, color, size: int = 200) -> None:
|
|
"""Write a solid-color RGB PNG using only the stdlib (no PIL needed)."""
|
|
raw = b"".join(b"\x00" + bytes(color) * size for _ in range(size))
|
|
|
|
def chunk(tag: bytes, data: bytes) -> bytes:
|
|
return (
|
|
struct.pack(">I", len(data))
|
|
+ tag
|
|
+ data
|
|
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
)
|
|
|
|
ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)
|
|
path.write_bytes(
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", ihdr)
|
|
+ chunk(b"IDAT", zlib.compress(raw))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def parse_created_chat_id(out: str) -> str | None:
|
|
m = re.search(r"== channel created: (\S+)", out)
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def sweep_leftovers(env, url, token) -> None:
|
|
"""Remove e2e-* channels / cron jobs left behind by earlier runs."""
|
|
rc, out, _ = run_probe(env, url, token, "--channel-list")
|
|
if rc == 0:
|
|
for m in re.finditer(r"== channel: (\S+) name='(e2e-[^']*)'", out):
|
|
chat_id, name = m.group(1), m.group(2)
|
|
print(f" cleanup: removing leftover channel {chat_id} ({name})")
|
|
run_probe(env, url, token, "--channel-delete", chat_id)
|
|
rc, out, _ = run_hermes(env, "cron", "list")
|
|
if rc == 0:
|
|
for m in re.finditer(
|
|
r"(\S+) \[(?:active|paused)\]\s*\n\s*Name:\s+(e2e-cron-[^ \n]*)", out
|
|
):
|
|
job_id, name = m.group(1), m.group(2)
|
|
print(f" cleanup: removing leftover cron job {job_id} ({name})")
|
|
run_hermes(env, "cron", "remove", job_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenarios (docs/13-testing.md §13.4)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def s1_pair(env, url, token):
|
|
rc, _, _ = run_probe(env, url, "definitely-wrong-token", "--authfail", "--send", "")
|
|
if rc != 0:
|
|
return FAIL, f"wrong token was not rejected (rc={rc})"
|
|
rc, _, _ = run_probe(env, url, token, "--send", "")
|
|
if rc != 0:
|
|
return FAIL, f"valid token did not pair (rc={rc})"
|
|
return PASS, "wrong token rejected; hello.ack on valid token"
|
|
|
|
|
|
def s2_text(env, url, token):
|
|
prompt = "Write a short poem about the ocean, at least 8 lines"
|
|
rc, _, _ = run_probe(
|
|
env, url, token, "--send", prompt, "--assert-turn", "--timeout", "120"
|
|
)
|
|
if rc == 0:
|
|
return PASS, "message.start -> >=1 message.update -> message.stop"
|
|
if rc == 10:
|
|
return FAIL, "no ordered start/update/stop segment"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s3_reasoning(env, url, token):
|
|
prompt = "Work out step by step: what is 17 * 23? Show your reasoning."
|
|
rc, _, _ = run_probe(
|
|
env, url, token, "--send", prompt, "--assert-reasoning", "--timeout", "120"
|
|
)
|
|
if rc == 0:
|
|
return PASS, "final message.stop carries non-empty reasoning"
|
|
if rc == 11:
|
|
return SKIP, "model returned no reasoning (model-dependent)"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s4_tools(env, url, token):
|
|
prompt = (
|
|
"List the files in your current working directory using your "
|
|
"shell tool, then tell me how many there are"
|
|
)
|
|
rc, _, _ = run_probe(
|
|
env, url, token, "--send", prompt, "--assert-tools", "--timeout", "150"
|
|
)
|
|
if rc == 0:
|
|
return PASS, "tool.start with a matching tool.end"
|
|
if rc == 12:
|
|
return FAIL, "no tool.start/tool.end pair"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s5_commentary(env, url, token):
|
|
prompt = (
|
|
"Research task: (1) use your shell tool to list the top-level "
|
|
"directories in /tmp, (2) report your findings so far, "
|
|
"(3) use your shell tool to count files in /tmp, "
|
|
"(4) report those findings too, (5) give a final summary of both"
|
|
)
|
|
rc, _, _ = run_probe(
|
|
env, url, token, "--send", prompt, "--assert-commentary", "--timeout", "150"
|
|
)
|
|
if rc == 0:
|
|
return PASS, "commentary frame observed"
|
|
if rc == 13:
|
|
return SKIP, "no commentary (model/agent-dependent per M2)"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s6_channels(env, url, token):
|
|
name = f"e2e-chan-{uuid.uuid4().hex[:6]}"
|
|
rc, out, _ = run_probe(env, url, token, "--channel-create", name)
|
|
if rc != 0:
|
|
return FAIL, f"channel.create failed (rc={rc})"
|
|
chat_id = parse_created_chat_id(out)
|
|
if not chat_id:
|
|
return FAIL, "channel.created received but chat_id not parseable"
|
|
rc, _, _ = run_probe(env, url, token, "--channel-delete", chat_id)
|
|
if rc != 0:
|
|
run_probe(env, url, token, "--channel-delete", chat_id) # best-effort
|
|
return FAIL, f"channel.delete failed (rc={rc})"
|
|
return PASS, f"created {chat_id} + deleted (cleanup)"
|
|
|
|
|
|
def s7_cron(env, url, token):
|
|
chan_name = f"e2e-cron-chan-{uuid.uuid4().hex[:6]}"
|
|
rc, out, _ = run_probe(env, url, token, "--channel-create", chan_name)
|
|
if rc != 0:
|
|
return SKIP, f"could not create cron target channel (rc={rc})"
|
|
chat_id = parse_created_chat_id(out)
|
|
if not chat_id:
|
|
return FAIL, "channel.created received but chat_id not parseable"
|
|
job_name = f"e2e-cron-{uuid.uuid4().hex[:6]}"
|
|
deliver = f"iris:{chat_id}"
|
|
rc, out, err = run_hermes(
|
|
env,
|
|
"cron",
|
|
"create",
|
|
"1m",
|
|
"Reply with exactly: e2e cron delivery OK",
|
|
"--deliver",
|
|
deliver,
|
|
"--name",
|
|
job_name,
|
|
)
|
|
job_id = None
|
|
if rc == 0:
|
|
m = re.search(r"Created job: (\S+)", out)
|
|
job_id = m.group(1) if m else None
|
|
try:
|
|
if rc != 0:
|
|
return SKIP, f"hermes cron create failed: {(err or out).strip()[:120]}"
|
|
rc, out, _ = run_probe(
|
|
env, url, token, "--watch", chat_id, "--timeout", "330", timeout=400
|
|
)
|
|
if rc == 0:
|
|
return PASS, f"one-shot cron job fired; message landed in {chat_id}"
|
|
return FAIL, f"no message in {chat_id} within 330s (probe rc={rc})"
|
|
finally:
|
|
if job_id:
|
|
run_hermes(env, "cron", "remove", job_id)
|
|
else:
|
|
# create succeeded but the id was not parseable: find by name.
|
|
_, list_out, _ = run_hermes(env, "cron", "list")
|
|
m = re.search(
|
|
r"(\S+) \[active\]\s*\n\s*Name:\s+" + re.escape(job_name), list_out
|
|
)
|
|
if m:
|
|
run_hermes(env, "cron", "remove", m.group(1))
|
|
run_probe(env, url, token, "--channel-delete", chat_id)
|
|
|
|
|
|
def s8_search(env, url, token):
|
|
marker = f"e2emarker{uuid.uuid4().hex[:8]}"
|
|
rc, _, _ = run_probe(
|
|
env,
|
|
url,
|
|
token,
|
|
"--send",
|
|
f"Remember this marker phrase: {marker}. Just acknowledge it briefly.",
|
|
"--timeout",
|
|
"120",
|
|
)
|
|
if rc != 0:
|
|
return FAIL, f"setup message failed (rc={rc})"
|
|
rc, _, _ = run_probe(env, url, token, "--send", "", "--search", marker)
|
|
if rc == 0:
|
|
return PASS, f"search for {marker!r} returned >=1 hit"
|
|
if rc == 14:
|
|
return FAIL, f"search for {marker!r} returned 0 hits"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s9_media_in(env, url, token):
|
|
png = Path(f"/tmp/e2e_in_{uuid.uuid4().hex[:6]}.png")
|
|
write_png(png, (30, 120, 220))
|
|
try:
|
|
rc, _, _ = run_probe(
|
|
env,
|
|
url,
|
|
token,
|
|
"--upload",
|
|
str(png),
|
|
"--send",
|
|
"describe this image briefly",
|
|
"--timeout",
|
|
"120",
|
|
)
|
|
if rc == 0:
|
|
return PASS, "upload + vision reply (final message)"
|
|
if rc == 8:
|
|
return FAIL, "media upload failed"
|
|
if rc == 7:
|
|
return FAIL, "no final message after upload"
|
|
return FAIL, f"probe rc={rc}"
|
|
finally:
|
|
png.unlink(missing_ok=True)
|
|
|
|
|
|
def s10_media_out(env, url, token):
|
|
prompt = (
|
|
"Create a 100x100 orange square PNG in /tmp with your tools. "
|
|
"In your final reply, include the MEDIA:/absolute/path tag for "
|
|
"that file so it is delivered to me."
|
|
)
|
|
rc, out, _ = run_probe(
|
|
env, url, token, "--send", prompt, "--pull-offer", "--timeout", "150"
|
|
)
|
|
m = re.search(r"== pulled (\d+) bytes", out)
|
|
if rc == 0 and m and int(m.group(1)) > 0:
|
|
return PASS, f"media.offer pulled ({m.group(1)} bytes)"
|
|
if rc == 9:
|
|
return FAIL, "media pull failed"
|
|
return FAIL, "no media.offer pulled (agent did not deliver an image)"
|
|
|
|
|
|
def s11_push(env, url, token):
|
|
rc, out, _ = run_probe(
|
|
env, url, token, "--fcm-token", "test-token-123", "--fcm-reg", "--send", ""
|
|
)
|
|
if rc != 0:
|
|
return FAIL, f"probe rc={rc}"
|
|
if "<- error" in out:
|
|
return FAIL, "error frame after fcm.register"
|
|
return PARTIAL, (
|
|
"fcm.register accepted (no error frame); device-notification leg is manual"
|
|
)
|
|
|
|
|
|
def s12_sync(env, url, token):
|
|
rc, out, _ = run_probe(env, url, token, "--sync", "0")
|
|
if rc == 0 and "sync done" in out:
|
|
return PARTIAL, (
|
|
"sync replay + sync.done verified; gateway-kill/restart leg is manual"
|
|
)
|
|
if rc == 8:
|
|
return FAIL, "sync failed"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
def s13_http_fallback(env, url, token):
|
|
"""docs/19: the HTTP fallback leg. The probe drives a full turn over
|
|
health + POST /v1/frame + SSE /v1/events (no WS involved). The user echo
|
|
must land on the SSE stream promptly after the POST (< 1.5 s on LAN)."""
|
|
u = urlparse(url)
|
|
scheme = "https" if u.scheme in ("wss", "https") else "http"
|
|
http_port = u.port or int(os.getenv("IRIS_HTTP_PORT", "8791"))
|
|
http_url = f"{scheme}://{u.hostname or '127.0.0.1'}:{http_port}"
|
|
rc, out, _ = run_probe(
|
|
env,
|
|
url,
|
|
token,
|
|
"--http",
|
|
"--http-url",
|
|
http_url,
|
|
"--send",
|
|
"Reply with exactly: e2e http fallback OK",
|
|
"--timeout",
|
|
"120",
|
|
)
|
|
if rc == 0:
|
|
m = re.search(r"== user echo in ([\d.]+)s", out)
|
|
echo = float(m.group(1)) if m else None
|
|
if echo is not None and echo > 1.5:
|
|
return FAIL, f"user echo took {echo:.2f}s (> 1.5 s)"
|
|
return PASS, (
|
|
"health + POST /v1/frame + SSE turn complete"
|
|
+ (f"; user echo in {echo:.2f}s" if echo is not None else "")
|
|
)
|
|
if rc == 20:
|
|
return FAIL, "health check failed (HTTP leg not running?)"
|
|
if rc == 21:
|
|
return FAIL, "SSE open failed"
|
|
if rc == 22:
|
|
return FAIL, "POST /v1/frame rejected"
|
|
return FAIL, f"probe rc={rc}"
|
|
|
|
|
|
SCENARIOS = [
|
|
(1, "pair", s1_pair),
|
|
(2, "text round-trip", s2_text),
|
|
(3, "reasoning", s3_reasoning),
|
|
(4, "tools", s4_tools),
|
|
(5, "commentary", s5_commentary),
|
|
(6, "channels", s6_channels),
|
|
(7, "cron delivery", s7_cron),
|
|
(8, "search", s8_search),
|
|
(9, "media in", s9_media_in),
|
|
(10, "media out", s10_media_out),
|
|
(11, "push", s11_push),
|
|
(12, "reconnect/sync", s12_sync),
|
|
(13, "http fallback", s13_http_fallback),
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser(
|
|
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
|
)
|
|
p.add_argument("--url", default=os.getenv("IRIS_HTTP_URL", DEFAULT_URL))
|
|
p.add_argument("--token", default="")
|
|
p.add_argument(
|
|
"--skip",
|
|
default="",
|
|
help="comma-separated scenario numbers to skip (e.g. 3,5,7)",
|
|
)
|
|
args = p.parse_args()
|
|
|
|
token = find_token(args.token)
|
|
if not token:
|
|
print("!! IRIS_TOKEN not found (env, hermes-agent/.env, or ~/.hermes/.env)")
|
|
return 1
|
|
skip = {int(x) for x in args.skip.split(",") if x.strip()}
|
|
|
|
env = dict(os.environ)
|
|
env["IRIS_TOKEN"] = token
|
|
|
|
print(f"== e2e: url={args.url} token={token[:6]}…")
|
|
sweep_leftovers(env, args.url, token)
|
|
|
|
results = []
|
|
for num, name, fn in SCENARIOS:
|
|
if num in skip:
|
|
results.append((num, name, SKIP, "skipped by --skip"))
|
|
print(f"[{num:2d}] {name:<18} {SKIP:<7} skipped by --skip")
|
|
continue
|
|
print(f"[{num:2d}] {name:<18} running…", flush=True)
|
|
try:
|
|
status, reason = fn(env, args.url, token)
|
|
except Exception as e:
|
|
status, reason = FAIL, f"driver error: {e}"
|
|
results.append((num, name, status, reason))
|
|
print(f"[{num:2d}] {name:<18} {status:<7} {reason}")
|
|
|
|
print()
|
|
print("=" * 78)
|
|
print(f"{'#':<3} {'scenario':<18} {'status':<8} reason")
|
|
print("-" * 78)
|
|
for num, name, status, reason in results:
|
|
print(f"{num:<3} {name:<18} {status:<8} {reason}")
|
|
print("-" * 78)
|
|
counts = {
|
|
s: sum(1 for r in results if r[2] == s) for s in (PASS, PARTIAL, SKIP, FAIL)
|
|
}
|
|
print(
|
|
f"total: {len(results)} PASS={counts[PASS]} PARTIAL={counts[PARTIAL]} "
|
|
f"SKIP={counts[SKIP]} FAIL={counts[FAIL]}"
|
|
)
|
|
return 1 if counts[FAIL] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|