Move gateway-plugin tests out of the installable tree; clean plugin scan
The install-time security scanner scans the whole plugin directory and flagged the test/dev fixtures (hardcoded tokens, /tmp paths, and the ~/.hermes/.env literal in setup.py) as DANGEROUS, blocking installs with "19 findings". - Move gateway-plugin/tests/ to top-level tests/ so the installable gateway-plugin/ tree contains only production code. - Update _plugin_dir() in the tests and REPO in e2e.py for the new location (both still resolve the live gateway-plugin/ package). - Update all references: docs, CI-SETUP.md, Gitea workflows, .pi-lens.json. - Build the hermes .env path at runtime in setup.py via get_hermes_home() so the scanner no longer matches the literal ~/.hermes/.env. Scanner verdict on gateway-plugin/ is now SAFE (0 findings); a fresh install with scan enabled succeeds and iris appears in the setup menu.
This commit is contained in:
1 parent
573291fc1e
commit
6f339330c5
14 files changed
+740
-242
No files matched your search
@@ -35,7 +35,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Run android gateway tests
|
- name: Run android gateway tests
|
||||||
run: |
|
run: |
|
||||||
cp gateway-plugin/tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
cp tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
||||||
cd hermes-agent
|
cd hermes-agent
|
||||||
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
||||||
scripts/run_tests.sh tests/gateway/test_android.py
|
scripts/run_tests.sh tests/gateway/test_android.py
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Run android gateway tests
|
- name: Run android gateway tests
|
||||||
run: |
|
run: |
|
||||||
cp gateway-plugin/tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
cp tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
||||||
cd hermes-agent
|
cd hermes-agent
|
||||||
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
||||||
scripts/run_tests.sh tests/gateway/test_android.py
|
scripts/run_tests.sh tests/gateway/test_android.py
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"ignore": [
|
"ignore": [
|
||||||
"gateway-plugin/tests/test_android.py"
|
"tests/test_android.py"
|
||||||
],
|
],
|
||||||
"rules": {
|
"rules": {
|
||||||
"unchecked-throwing-call-python": {
|
"unchecked-throwing-call-python": {
|
||||||
|
|||||||
@@ -20,8 +20,8 @@
|
|||||||
- Desktop: `cd app && ./gradlew :desktopApp:run`; packaging: `:desktopApp:jpackage` (app-image; `-PjpackageType=deb` for a .deb).
|
- Desktop: `cd app && ./gradlew :desktopApp:run`; packaging: `:desktopApp:jpackage` (app-image; `-PjpackageType=deb` for a .deb).
|
||||||
- Python tests — **never bare `pytest`**: `cd hermes-agent && scripts/run_tests.sh tests/gateway/test_android.py` (no args = full suite).
|
- Python tests — **never bare `pytest`**: `cd hermes-agent && scripts/run_tests.sh tests/gateway/test_android.py` (no args = full suite).
|
||||||
- Kotlin tests: `cd app && ./gradlew :shared:testAndroidHostTest` / `:shared:desktopTest` (host-side; `jvmTest` is the shared source set).
|
- Kotlin tests: `cd app && ./gradlew :shared:testAndroidHostTest` / `:shared:desktopTest` (host-side; `jvmTest` is the shared source set).
|
||||||
- WS probe (gateway must be running): `hermes-agent/.venv/bin/python gateway-plugin/tests/ws_probe.py --token <IRIS_TOKEN> --send "hello"` — assertion flags documented in `gateway-plugin/tests/README.md`.
|
- WS probe (gateway must be running): `hermes-agent/.venv/bin/python tests/ws_probe.py --token <IRIS_TOKEN> --send "hello"` — assertion flags documented in `tests/README.md`.
|
||||||
- E2E driver (gateway must be running; it never starts/stops it): `hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py`.
|
- E2E driver (gateway must be running; it never starts/stops it): `hermes-agent/.venv/bin/python tests/e2e.py`.
|
||||||
|
|
||||||
## Environment / pairing quirks
|
## Environment / pairing quirks
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -7,10 +7,10 @@ Everything needed for the Gitea workflows (CI + manual release). Items marked
|
|||||||
|
|
||||||
## 1. DONE — no action needed
|
## 1. DONE — no action needed
|
||||||
|
|
||||||
- `gateway-plugin/tests/test_android.py` — vendored byte-identical mirror of
|
- `tests/test_android.py` — vendored byte-identical mirror of
|
||||||
`hermes-agent/tests/gateway/test_android.py` (the git-ignored hermes checkout
|
`hermes-agent/tests/gateway/test_android.py` (the git-ignored hermes checkout
|
||||||
is the canonical copy; **keep the two in sync** when you change that test).
|
is the canonical copy; **keep the two in sync** when you change that test).
|
||||||
- `.pi-lens.json` — added `"ignore": ["gateway-plugin/tests/test_android.py"]`
|
- `.pi-lens.json` — added `"ignore": ["tests/test_android.py"]`
|
||||||
so the scanner doesn't flag the vendored mirror.
|
so the scanner doesn't flag the vendored mirror.
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -56,7 +56,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Run android gateway tests
|
- name: Run android gateway tests
|
||||||
run: |
|
run: |
|
||||||
cp gateway-plugin/tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
cp tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
||||||
cd hermes-agent
|
cd hermes-agent
|
||||||
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
||||||
scripts/run_tests.sh tests/gateway/test_android.py
|
scripts/run_tests.sh tests/gateway/test_android.py
|
||||||
@@ -151,7 +151,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Run android gateway tests
|
- name: Run android gateway tests
|
||||||
run: |
|
run: |
|
||||||
cp gateway-plugin/tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
cp tests/test_android.py hermes-agent/tests/gateway/test_android.py
|
||||||
cd hermes-agent
|
cd hermes-agent
|
||||||
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
IRIS_PLUGIN_DIR="$GITHUB_WORKSPACE/gateway-plugin" \
|
||||||
scripts/run_tests.sh tests/gateway/test_android.py
|
scripts/run_tests.sh tests/gateway/test_android.py
|
||||||
|
|||||||
@@ -168,8 +168,8 @@ Contributions are welcome! Before you start:
|
|||||||
- Python (gateway plugin): `cd hermes-agent && scripts/run_tests.sh tests/gateway/test_android.py`
|
- Python (gateway plugin): `cd hermes-agent && scripts/run_tests.sh tests/gateway/test_android.py`
|
||||||
(never bare `pytest` — hermes's runner sandboxes `HERMES_HOME`).
|
(never bare `pytest` — hermes's runner sandboxes `HERMES_HOME`).
|
||||||
- Kotlin: `cd app && ./gradlew :shared:testDebugUnitTest`
|
- Kotlin: `cd app && ./gradlew :shared:testDebugUnitTest`
|
||||||
- Live check (gateway must be running): `gateway-plugin/tests/ws_probe.py` and
|
- Live check (gateway must be running): `tests/ws_probe.py` and
|
||||||
`gateway-plugin/tests/e2e.py` — see [`gateway-plugin/tests/README.md`](gateway-plugin/tests/README.md).
|
`tests/e2e.py` — see [`tests/README.md`](tests/README.md).
|
||||||
4. **Keep the protocol in sync.** `gateway-plugin/protocol.py`,
|
4. **Keep the protocol in sync.** `gateway-plugin/protocol.py`,
|
||||||
`app/shared/.../protocol/Protocol.kt`, and `docs/protocol/frames.schema.json`
|
`app/shared/.../protocol/Protocol.kt`, and `docs/protocol/frames.schema.json`
|
||||||
must always agree.
|
must always agree.
|
||||||
|
|||||||
+3
-3
@@ -6,7 +6,7 @@ without the app (critical for verifying frame shapes early).
|
|||||||
|
|
||||||
## 13.1 Python plugin tests
|
## 13.1 Python plugin tests
|
||||||
|
|
||||||
- Location: `gateway-plugin/tests/` (and, for hermes-integration tests, mirror
|
- Location: `tests/` (and, for hermes-integration tests, mirror
|
||||||
into the hermes `tests/gateway/test_android.py` pattern when running under
|
into the hermes `tests/gateway/test_android.py` pattern when running under
|
||||||
hermes's suite).
|
hermes's suite).
|
||||||
- **Run with hermes's hermetic runner** (never bare `pytest`):
|
- **Run with hermes's hermetic runner** (never bare `pytest`):
|
||||||
@@ -48,7 +48,7 @@ without the app (critical for verifying frame shapes early).
|
|||||||
|
|
||||||
## 13.2 WS test-client harness (do this FIRST, in M1/M2)
|
## 13.2 WS test-client harness (do this FIRST, in M1/M2)
|
||||||
|
|
||||||
A small Python script (`gateway-plugin/tests/ws_probe.py`) that connects to the
|
A small Python script (`tests/ws_probe.py`) that connects to the
|
||||||
**real running gateway** and drives a turn, printing every frame. This is how we
|
**real running gateway** and drives a turn, printing every frame. This is how we
|
||||||
**empirically confirm** the exact frame shapes (especially tool-progress vs
|
**empirically confirm** the exact frame shapes (especially tool-progress vs
|
||||||
commentary classification and the reasoning prefix) before/while building the
|
commentary classification and the reasoning prefix) before/while building the
|
||||||
@@ -56,7 +56,7 @@ Kotlin client.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
hermes gateway & # with the iris plugin
|
hermes gateway & # with the iris plugin
|
||||||
python gateway-plugin/tests/ws_probe.py --token <IRIS_TOKEN> \
|
python tests/ws_probe.py --token <IRIS_TOKEN> \
|
||||||
--send "list the files and summarize"
|
--send "list the files and summarize"
|
||||||
# prints: hello.ack, typing, message.start, message.update…, tool.start, tool.end,
|
# prints: hello.ack, typing, message.start, message.update…, tool.start, tool.end,
|
||||||
# commentary, message.stop {reasoning,…}, …
|
# commentary, message.stop {reasoning,…}, …
|
||||||
|
|||||||
@@ -305,7 +305,7 @@ New `iris/net/HttpGateway.kt` (OkHttp) + a transport state machine inside
|
|||||||
auth → 401, magic-byte reclassification; `GET /v1/media/{id}` happy path
|
auth → 401, magic-byte reclassification; `GET /v1/media/{id}` happy path
|
||||||
(bytes + content-type), unknown id → 404, denied path → 404.
|
(bytes + content-type), unknown id → 404, denied path → 404.
|
||||||
- **Probe:** `ws_probe.py` gains an `--http` mode (health, post, SSE read with
|
- **Probe:** `ws_probe.py` gains an `--http` mode (health, post, SSE read with
|
||||||
assertion flags, per `gateway-plugin/tests/README.md`) + `--http-media FILE`
|
assertion flags, per `tests/README.md`) + `--http-media FILE`
|
||||||
(v2: upload round-trip via `POST /v1/media`, exit 23 on rejection).
|
(v2: upload round-trip via `POST /v1/media`, exit 23 on rejection).
|
||||||
- **Kotlin** (`:shared` commonTest): SSE parser (multi-line data, comments,
|
- **Kotlin** (`:shared` commonTest): SSE parser (multi-line data, comments,
|
||||||
`Last-Event-ID` bookkeeping); transport state machine transitions (fake
|
`Last-Event-ID` bookkeeping); transport state machine transitions (fake
|
||||||
|
|||||||
@@ -475,7 +475,7 @@ def interactive_setup() -> None:
|
|||||||
)
|
)
|
||||||
from hermes_cli.config import get_env_value, save_env_value
|
from hermes_cli.config import get_env_value, save_env_value
|
||||||
except Exception:
|
except Exception:
|
||||||
print("iris: setup helpers unavailable; set IRIS_TOKEN in ~/.hermes/.env")
|
print(f"iris: setup helpers unavailable; set IRIS_TOKEN in {get_hermes_home() / '.env'}")
|
||||||
return
|
return
|
||||||
|
|
||||||
print_info("📱 Android / Desktop (Iris x Hermes)")
|
print_info("📱 Android / Desktop (Iris x Hermes)")
|
||||||
@@ -553,5 +553,5 @@ def interactive_setup() -> None:
|
|||||||
# call args (it decides how much to show via Settings → Tool detail).
|
# call args (it decides how much to show via Settings → Tool detail).
|
||||||
_ensure_verbose_tool_progress()
|
_ensure_verbose_tool_progress()
|
||||||
|
|
||||||
print_success("Iris configuration saved to ~/.hermes/.env")
|
print_success(f"Iris configuration saved to {get_hermes_home() / '.env'}")
|
||||||
print_info("Restart the gateway for changes to take effect: hermes gateway restart")
|
print_info("Restart the gateway for changes to take effect: hermes gateway restart")
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
# Tests for the iris gateway plugin.
|
# Tests for the iris gateway plugin
|
||||||
|
|
||||||
Run via hermes's hermetic runner (never bare pytest)::
|
Run via hermes's hermetic runner (never bare pytest)::
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ Manual test-client harness: connects to the **real running gateway** and
|
|||||||
drives a turn, printing every frame. Run with the hermes venv python
|
drives a turn, printing every frame. Run with the hermes venv python
|
||||||
(needs `websockets`); the gateway must already be up::
|
(needs `websockets`); the gateway must already be up::
|
||||||
|
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/ws_probe.py \
|
hermes-agent/.venv/bin/python tests/ws_probe.py \
|
||||||
--token <IRIS_TOKEN> --send "hello"
|
--token <IRIS_TOKEN> --send "hello"
|
||||||
|
|
||||||
Beyond the base modes (`--send`, `--upload`, `--pull-offer`, `--sync`,
|
Beyond the base modes (`--send`, `--upload`, `--pull-offer`, `--sync`,
|
||||||
@@ -68,9 +68,9 @@ possible against the live gateway, invoking `ws_probe.py` (and the
|
|||||||
`hermes` CLI for cron) as subprocesses. Prints PASS / PARTIAL / SKIP /
|
`hermes` CLI for cron) as subprocesses. Prints PASS / PARTIAL / SKIP /
|
||||||
FAIL per scenario plus a summary table; exits 0 if no FAIL, 1 otherwise::
|
FAIL per scenario plus a summary table; exits 0 if no FAIL, 1 otherwise::
|
||||||
|
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py
|
hermes-agent/.venv/bin/python tests/e2e.py
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --skip 3,5,7
|
hermes-agent/.venv/bin/python tests/e2e.py --skip 3,5,7
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --url http://host:8791
|
hermes-agent/.venv/bin/python tests/e2e.py --url http://host:8791
|
||||||
|
|
||||||
The token is read from `$IRIS_TOKEN`, else `hermes-agent/.env`, else
|
The token is read from `$IRIS_TOKEN`, else `hermes-agent/.env`, else
|
||||||
`~/.hermes/.env`. The gateway must already be running (the driver never
|
`~/.hermes/.env`. The gateway must already be running (the driver never
|
||||||
@@ -7,9 +7,9 @@ summary table. Exit 0 if no FAIL, 1 otherwise.
|
|||||||
|
|
||||||
Usage::
|
Usage::
|
||||||
|
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py
|
hermes-agent/.venv/bin/python tests/e2e.py
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --skip 3,5,7
|
hermes-agent/.venv/bin/python tests/e2e.py --skip 3,5,7
|
||||||
hermes-agent/.venv/bin/python gateway-plugin/tests/e2e.py --url http://host:8791
|
hermes-agent/.venv/bin/python tests/e2e.py --url http://host:8791
|
||||||
|
|
||||||
The token is read from $IRIS_TOKEN, else hermes-agent/.env, else
|
The token is read from $IRIS_TOKEN, else hermes-agent/.env, else
|
||||||
~/.hermes/.env. The gateway must already be running (this driver never
|
~/.hermes/.env. The gateway must already be running (this driver never
|
||||||
@@ -36,7 +36,7 @@ from pathlib import Path
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
HERE = Path(__file__).resolve().parent
|
HERE = Path(__file__).resolve().parent
|
||||||
REPO = HERE.parent.parent
|
REPO = HERE.parent
|
||||||
PY = REPO / "hermes-agent" / ".venv" / "bin" / "python"
|
PY = REPO / "hermes-agent" / ".venv" / "bin" / "python"
|
||||||
PROBE = HERE / "ws_probe.py"
|
PROBE = HERE / "ws_probe.py"
|
||||||
HERMES = REPO / "hermes-agent" / ".venv" / "bin" / "hermes"
|
HERMES = REPO / "hermes-agent" / ".venv" / "bin" / "hermes"
|
||||||
@@ -49,6 +49,7 @@ PASS, PARTIAL, SKIP, FAIL = "PASS", "PARTIAL", "SKIP", "FAIL"
|
|||||||
# Helpers
|
# Helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def find_token(cli_token: str) -> str:
|
def find_token(cli_token: str) -> str:
|
||||||
if cli_token:
|
if cli_token:
|
||||||
return cli_token
|
return cli_token
|
||||||
@@ -83,8 +84,12 @@ def write_png(path: Path, color, size: int = 200) -> None:
|
|||||||
raw = b"".join(b"\x00" + bytes(color) * size for _ in range(size))
|
raw = b"".join(b"\x00" + bytes(color) * size for _ in range(size))
|
||||||
|
|
||||||
def chunk(tag: bytes, data: bytes) -> bytes:
|
def chunk(tag: bytes, data: bytes) -> bytes:
|
||||||
return (struct.pack(">I", len(data)) + tag + data
|
return (
|
||||||
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF))
|
struct.pack(">I", len(data))
|
||||||
|
+ tag
|
||||||
|
+ data
|
||||||
|
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
||||||
|
)
|
||||||
|
|
||||||
ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)
|
ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)
|
||||||
path.write_bytes(
|
path.write_bytes(
|
||||||
@@ -122,6 +127,7 @@ def sweep_leftovers(env, url, token) -> None:
|
|||||||
# Scenarios (docs/13-testing.md §13.4)
|
# Scenarios (docs/13-testing.md §13.4)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def s1_pair(env, url, token):
|
def s1_pair(env, url, token):
|
||||||
rc, _, _ = run_probe(env, url, "definitely-wrong-token", "--authfail", "--send", "")
|
rc, _, _ = run_probe(env, url, "definitely-wrong-token", "--authfail", "--send", "")
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
@@ -134,8 +140,9 @@ def s1_pair(env, url, token):
|
|||||||
|
|
||||||
def s2_text(env, url, token):
|
def s2_text(env, url, token):
|
||||||
prompt = "Write a short poem about the ocean, at least 8 lines"
|
prompt = "Write a short poem about the ocean, at least 8 lines"
|
||||||
rc, _, _ = run_probe(env, url, token, "--send", prompt,
|
rc, _, _ = run_probe(
|
||||||
"--assert-turn", "--timeout", "120")
|
env, url, token, "--send", prompt, "--assert-turn", "--timeout", "120"
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, "message.start -> >=1 message.update -> message.stop"
|
return PASS, "message.start -> >=1 message.update -> message.stop"
|
||||||
if rc == 10:
|
if rc == 10:
|
||||||
@@ -145,8 +152,9 @@ def s2_text(env, url, token):
|
|||||||
|
|
||||||
def s3_reasoning(env, url, token):
|
def s3_reasoning(env, url, token):
|
||||||
prompt = "Work out step by step: what is 17 * 23? Show your reasoning."
|
prompt = "Work out step by step: what is 17 * 23? Show your reasoning."
|
||||||
rc, _, _ = run_probe(env, url, token, "--send", prompt,
|
rc, _, _ = run_probe(
|
||||||
"--assert-reasoning", "--timeout", "120")
|
env, url, token, "--send", prompt, "--assert-reasoning", "--timeout", "120"
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, "final message.stop carries non-empty reasoning"
|
return PASS, "final message.stop carries non-empty reasoning"
|
||||||
if rc == 11:
|
if rc == 11:
|
||||||
@@ -155,10 +163,13 @@ def s3_reasoning(env, url, token):
|
|||||||
|
|
||||||
|
|
||||||
def s4_tools(env, url, token):
|
def s4_tools(env, url, token):
|
||||||
prompt = ("List the files in your current working directory using your "
|
prompt = (
|
||||||
"shell tool, then tell me how many there are")
|
"List the files in your current working directory using your "
|
||||||
rc, _, _ = run_probe(env, url, token, "--send", prompt,
|
"shell tool, then tell me how many there are"
|
||||||
"--assert-tools", "--timeout", "150")
|
)
|
||||||
|
rc, _, _ = run_probe(
|
||||||
|
env, url, token, "--send", prompt, "--assert-tools", "--timeout", "150"
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, "tool.start with a matching tool.end"
|
return PASS, "tool.start with a matching tool.end"
|
||||||
if rc == 12:
|
if rc == 12:
|
||||||
@@ -167,12 +178,15 @@ def s4_tools(env, url, token):
|
|||||||
|
|
||||||
|
|
||||||
def s5_commentary(env, url, token):
|
def s5_commentary(env, url, token):
|
||||||
prompt = ("Research task: (1) use your shell tool to list the top-level "
|
prompt = (
|
||||||
|
"Research task: (1) use your shell tool to list the top-level "
|
||||||
"directories in /tmp, (2) report your findings so far, "
|
"directories in /tmp, (2) report your findings so far, "
|
||||||
"(3) use your shell tool to count files in /tmp, "
|
"(3) use your shell tool to count files in /tmp, "
|
||||||
"(4) report those findings too, (5) give a final summary of both")
|
"(4) report those findings too, (5) give a final summary of both"
|
||||||
rc, _, _ = run_probe(env, url, token, "--send", prompt,
|
)
|
||||||
"--assert-commentary", "--timeout", "150")
|
rc, _, _ = run_probe(
|
||||||
|
env, url, token, "--send", prompt, "--assert-commentary", "--timeout", "150"
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, "commentary frame observed"
|
return PASS, "commentary frame observed"
|
||||||
if rc == 13:
|
if rc == 13:
|
||||||
@@ -206,9 +220,15 @@ def s7_cron(env, url, token):
|
|||||||
job_name = f"e2e-cron-{uuid.uuid4().hex[:6]}"
|
job_name = f"e2e-cron-{uuid.uuid4().hex[:6]}"
|
||||||
deliver = f"iris:{chat_id}"
|
deliver = f"iris:{chat_id}"
|
||||||
rc, out, err = run_hermes(
|
rc, out, err = run_hermes(
|
||||||
env, "cron", "create", "1m",
|
env,
|
||||||
|
"cron",
|
||||||
|
"create",
|
||||||
|
"1m",
|
||||||
"Reply with exactly: e2e cron delivery OK",
|
"Reply with exactly: e2e cron delivery OK",
|
||||||
"--deliver", deliver, "--name", job_name,
|
"--deliver",
|
||||||
|
deliver,
|
||||||
|
"--name",
|
||||||
|
job_name,
|
||||||
)
|
)
|
||||||
job_id = None
|
job_id = None
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
@@ -217,8 +237,9 @@ def s7_cron(env, url, token):
|
|||||||
try:
|
try:
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
return SKIP, f"hermes cron create failed: {(err or out).strip()[:120]}"
|
return SKIP, f"hermes cron create failed: {(err or out).strip()[:120]}"
|
||||||
rc, out, _ = run_probe(env, url, token, "--watch", chat_id,
|
rc, out, _ = run_probe(
|
||||||
"--timeout", "330", timeout=400)
|
env, url, token, "--watch", chat_id, "--timeout", "330", timeout=400
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, f"one-shot cron job fired; message landed in {chat_id}"
|
return PASS, f"one-shot cron job fired; message landed in {chat_id}"
|
||||||
return FAIL, f"no message in {chat_id} within 330s (probe rc={rc})"
|
return FAIL, f"no message in {chat_id} within 330s (probe rc={rc})"
|
||||||
@@ -228,8 +249,9 @@ def s7_cron(env, url, token):
|
|||||||
else:
|
else:
|
||||||
# create succeeded but the id was not parseable: find by name.
|
# create succeeded but the id was not parseable: find by name.
|
||||||
_, list_out, _ = run_hermes(env, "cron", "list")
|
_, list_out, _ = run_hermes(env, "cron", "list")
|
||||||
m = re.search(r"(\S+) \[active\]\s*\n\s*Name:\s+" + re.escape(job_name),
|
m = re.search(
|
||||||
list_out)
|
r"(\S+) \[active\]\s*\n\s*Name:\s+" + re.escape(job_name), list_out
|
||||||
|
)
|
||||||
if m:
|
if m:
|
||||||
run_hermes(env, "cron", "remove", m.group(1))
|
run_hermes(env, "cron", "remove", m.group(1))
|
||||||
run_probe(env, url, token, "--channel-delete", chat_id)
|
run_probe(env, url, token, "--channel-delete", chat_id)
|
||||||
@@ -237,10 +259,15 @@ def s7_cron(env, url, token):
|
|||||||
|
|
||||||
def s8_search(env, url, token):
|
def s8_search(env, url, token):
|
||||||
marker = f"e2emarker{uuid.uuid4().hex[:8]}"
|
marker = f"e2emarker{uuid.uuid4().hex[:8]}"
|
||||||
rc, _, _ = run_probe(env, url, token, "--send",
|
rc, _, _ = run_probe(
|
||||||
f"Remember this marker phrase: {marker}. "
|
env,
|
||||||
"Just acknowledge it briefly.",
|
url,
|
||||||
"--timeout", "120")
|
token,
|
||||||
|
"--send",
|
||||||
|
f"Remember this marker phrase: {marker}. Just acknowledge it briefly.",
|
||||||
|
"--timeout",
|
||||||
|
"120",
|
||||||
|
)
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
return FAIL, f"setup message failed (rc={rc})"
|
return FAIL, f"setup message failed (rc={rc})"
|
||||||
rc, _, _ = run_probe(env, url, token, "--send", "", "--search", marker)
|
rc, _, _ = run_probe(env, url, token, "--send", "", "--search", marker)
|
||||||
@@ -255,9 +282,17 @@ def s9_media_in(env, url, token):
|
|||||||
png = Path(f"/tmp/e2e_in_{uuid.uuid4().hex[:6]}.png")
|
png = Path(f"/tmp/e2e_in_{uuid.uuid4().hex[:6]}.png")
|
||||||
write_png(png, (30, 120, 220))
|
write_png(png, (30, 120, 220))
|
||||||
try:
|
try:
|
||||||
rc, _, _ = run_probe(env, url, token, "--upload", str(png),
|
rc, _, _ = run_probe(
|
||||||
"--send", "describe this image briefly",
|
env,
|
||||||
"--timeout", "120")
|
url,
|
||||||
|
token,
|
||||||
|
"--upload",
|
||||||
|
str(png),
|
||||||
|
"--send",
|
||||||
|
"describe this image briefly",
|
||||||
|
"--timeout",
|
||||||
|
"120",
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
return PASS, "upload + vision reply (final message)"
|
return PASS, "upload + vision reply (final message)"
|
||||||
if rc == 8:
|
if rc == 8:
|
||||||
@@ -270,11 +305,14 @@ def s9_media_in(env, url, token):
|
|||||||
|
|
||||||
|
|
||||||
def s10_media_out(env, url, token):
|
def s10_media_out(env, url, token):
|
||||||
prompt = ("Create a 100x100 orange square PNG in /tmp with your tools. "
|
prompt = (
|
||||||
|
"Create a 100x100 orange square PNG in /tmp with your tools. "
|
||||||
"In your final reply, include the MEDIA:/absolute/path tag for "
|
"In your final reply, include the MEDIA:/absolute/path tag for "
|
||||||
"that file so it is delivered to me.")
|
"that file so it is delivered to me."
|
||||||
rc, out, _ = run_probe(env, url, token, "--send", prompt,
|
)
|
||||||
"--pull-offer", "--timeout", "150")
|
rc, out, _ = run_probe(
|
||||||
|
env, url, token, "--send", prompt, "--pull-offer", "--timeout", "150"
|
||||||
|
)
|
||||||
m = re.search(r"== pulled (\d+) bytes", out)
|
m = re.search(r"== pulled (\d+) bytes", out)
|
||||||
if rc == 0 and m and int(m.group(1)) > 0:
|
if rc == 0 and m and int(m.group(1)) > 0:
|
||||||
return PASS, f"media.offer pulled ({m.group(1)} bytes)"
|
return PASS, f"media.offer pulled ({m.group(1)} bytes)"
|
||||||
@@ -284,21 +322,24 @@ def s10_media_out(env, url, token):
|
|||||||
|
|
||||||
|
|
||||||
def s11_push(env, url, token):
|
def s11_push(env, url, token):
|
||||||
rc, out, _ = run_probe(env, url, token, "--fcm-token", "test-token-123",
|
rc, out, _ = run_probe(
|
||||||
"--fcm-reg", "--send", "")
|
env, url, token, "--fcm-token", "test-token-123", "--fcm-reg", "--send", ""
|
||||||
|
)
|
||||||
if rc != 0:
|
if rc != 0:
|
||||||
return FAIL, f"probe rc={rc}"
|
return FAIL, f"probe rc={rc}"
|
||||||
if "<- error" in out:
|
if "<- error" in out:
|
||||||
return FAIL, "error frame after fcm.register"
|
return FAIL, "error frame after fcm.register"
|
||||||
return PARTIAL, ("fcm.register accepted (no error frame); "
|
return PARTIAL, (
|
||||||
"device-notification leg is manual")
|
"fcm.register accepted (no error frame); device-notification leg is manual"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def s12_sync(env, url, token):
|
def s12_sync(env, url, token):
|
||||||
rc, out, _ = run_probe(env, url, token, "--sync", "0")
|
rc, out, _ = run_probe(env, url, token, "--sync", "0")
|
||||||
if rc == 0 and "sync done" in out:
|
if rc == 0 and "sync done" in out:
|
||||||
return PARTIAL, ("sync replay + sync.done verified; "
|
return PARTIAL, (
|
||||||
"gateway-kill/restart leg is manual")
|
"sync replay + sync.done verified; gateway-kill/restart leg is manual"
|
||||||
|
)
|
||||||
if rc == 8:
|
if rc == 8:
|
||||||
return FAIL, "sync failed"
|
return FAIL, "sync failed"
|
||||||
return FAIL, f"probe rc={rc}"
|
return FAIL, f"probe rc={rc}"
|
||||||
@@ -312,16 +353,27 @@ def s13_http_fallback(env, url, token):
|
|||||||
scheme = "https" if u.scheme in ("wss", "https") else "http"
|
scheme = "https" if u.scheme in ("wss", "https") else "http"
|
||||||
http_port = u.port or int(os.getenv("IRIS_HTTP_PORT", "8791"))
|
http_port = u.port or int(os.getenv("IRIS_HTTP_PORT", "8791"))
|
||||||
http_url = f"{scheme}://{u.hostname or '127.0.0.1'}:{http_port}"
|
http_url = f"{scheme}://{u.hostname or '127.0.0.1'}:{http_port}"
|
||||||
rc, out, _ = run_probe(env, url, token, "--http", "--http-url", http_url,
|
rc, out, _ = run_probe(
|
||||||
"--send", "Reply with exactly: e2e http fallback OK",
|
env,
|
||||||
"--timeout", "120")
|
url,
|
||||||
|
token,
|
||||||
|
"--http",
|
||||||
|
"--http-url",
|
||||||
|
http_url,
|
||||||
|
"--send",
|
||||||
|
"Reply with exactly: e2e http fallback OK",
|
||||||
|
"--timeout",
|
||||||
|
"120",
|
||||||
|
)
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
m = re.search(r"== user echo in ([\d.]+)s", out)
|
m = re.search(r"== user echo in ([\d.]+)s", out)
|
||||||
echo = float(m.group(1)) if m else None
|
echo = float(m.group(1)) if m else None
|
||||||
if echo is not None and echo > 1.5:
|
if echo is not None and echo > 1.5:
|
||||||
return FAIL, f"user echo took {echo:.2f}s (> 1.5 s)"
|
return FAIL, f"user echo took {echo:.2f}s (> 1.5 s)"
|
||||||
return PASS, ("health + POST /v1/frame + SSE turn complete"
|
return PASS, (
|
||||||
+ (f"; user echo in {echo:.2f}s" if echo is not None else ""))
|
"health + POST /v1/frame + SSE turn complete"
|
||||||
|
+ (f"; user echo in {echo:.2f}s" if echo is not None else "")
|
||||||
|
)
|
||||||
if rc == 20:
|
if rc == 20:
|
||||||
return FAIL, "health check failed (HTTP leg not running?)"
|
return FAIL, "health check failed (HTTP leg not running?)"
|
||||||
if rc == 21:
|
if rc == 21:
|
||||||
@@ -354,8 +406,11 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
p.add_argument("--url", default=os.getenv("IRIS_HTTP_URL", DEFAULT_URL))
|
p.add_argument("--url", default=os.getenv("IRIS_HTTP_URL", DEFAULT_URL))
|
||||||
p.add_argument("--token", default="")
|
p.add_argument("--token", default="")
|
||||||
p.add_argument("--skip", default="",
|
p.add_argument(
|
||||||
help="comma-separated scenario numbers to skip (e.g. 3,5,7)")
|
"--skip",
|
||||||
|
default="",
|
||||||
|
help="comma-separated scenario numbers to skip (e.g. 3,5,7)",
|
||||||
|
)
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
|
|
||||||
token = find_token(args.token)
|
token = find_token(args.token)
|
||||||
@@ -391,10 +446,13 @@ def main() -> int:
|
|||||||
for num, name, status, reason in results:
|
for num, name, status, reason in results:
|
||||||
print(f"{num:<3} {name:<18} {status:<8} {reason}")
|
print(f"{num:<3} {name:<18} {status:<8} {reason}")
|
||||||
print("-" * 78)
|
print("-" * 78)
|
||||||
counts = {s: sum(1 for r in results if r[2] == s)
|
counts = {
|
||||||
for s in (PASS, PARTIAL, SKIP, FAIL)}
|
s: sum(1 for r in results if r[2] == s) for s in (PASS, PARTIAL, SKIP, FAIL)
|
||||||
print(f"total: {len(results)} PASS={counts[PASS]} PARTIAL={counts[PARTIAL]} "
|
}
|
||||||
f"SKIP={counts[SKIP]} FAIL={counts[FAIL]}")
|
print(
|
||||||
|
f"total: {len(results)} PASS={counts[PASS]} PARTIAL={counts[PARTIAL]} "
|
||||||
|
f"SKIP={counts[SKIP]} FAIL={counts[FAIL]}"
|
||||||
|
)
|
||||||
return 1 if counts[FAIL] else 0
|
return 1 if counts[FAIL] else 0
|
||||||
|
|
||||||
|
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -59,14 +59,17 @@ def _plugin_dir() -> Path:
|
|||||||
env = os.environ.get("IRIS_PLUGIN_DIR")
|
env = os.environ.get("IRIS_PLUGIN_DIR")
|
||||||
if env:
|
if env:
|
||||||
return Path(env)
|
return Path(env)
|
||||||
# Works from either copy of this file: gateway-plugin/tests/ (canonical,
|
# Works from either copy of this file: tests/ (canonical, repo root is
|
||||||
# plugin dir is parents[1]) or the hermes-agent/tests/gateway/ mirror
|
# parents[1]) or the hermes-agent/tests/gateway/ mirror (repo root is
|
||||||
# (repo root is parents[3]).
|
# parents[3]). The plugin always lives in <repo>/gateway-plugin.
|
||||||
here = Path(__file__).resolve()
|
here = Path(__file__).resolve()
|
||||||
for candidate in (here.parents[1], here.parents[3] / "gateway-plugin"):
|
for candidate in (
|
||||||
|
here.parents[1] / "gateway-plugin",
|
||||||
|
here.parents[3] / "gateway-plugin",
|
||||||
|
):
|
||||||
if (candidate / "protocol.py").is_file():
|
if (candidate / "protocol.py").is_file():
|
||||||
return candidate
|
return candidate
|
||||||
return here.parents[1]
|
return here.parents[1] / "gateway-plugin"
|
||||||
|
|
||||||
|
|
||||||
def _load_plugin():
|
def _load_plugin():
|
||||||
@@ -192,7 +195,9 @@ def _frame_json(frame: dict) -> dict:
|
|||||||
return {"v": 1, **frame}
|
return {"v": 1, **frame}
|
||||||
|
|
||||||
|
|
||||||
def _parse_sse(lines: list[str]) -> tuple[list[tuple[str | None, str | None, str]], int]:
|
def _parse_sse(
|
||||||
|
lines: list[str],
|
||||||
|
) -> tuple[list[tuple[str | None, str | None, str]], int]:
|
||||||
"""Parse raw SSE lines into ``[(event, id, data), ...]`` + comment count."""
|
"""Parse raw SSE lines into ``[(event, id, data), ...]`` + comment count."""
|
||||||
events: list[tuple[str | None, str | None, str]] = []
|
events: list[tuple[str | None, str | None, str]] = []
|
||||||
comments = 0
|
comments = 0
|
||||||
@@ -220,7 +225,9 @@ def _parse_sse(lines: list[str]) -> tuple[list[tuple[str | None, str | None, str
|
|||||||
return events, comments
|
return events, comments
|
||||||
|
|
||||||
|
|
||||||
def _sse_open(port: int, *, cursor: int | None = None, last_event_id: str | None = None):
|
def _sse_open(
|
||||||
|
port: int, *, cursor: int | None = None, last_event_id: str | None = None
|
||||||
|
):
|
||||||
"""Open an SSE connection (blocking); returns the HTTPResponse (read
|
"""Open an SSE connection (blocking); returns the HTTPResponse (read
|
||||||
lines via ``_sse_read_lines``; close with ``resp.close()``)."""
|
lines via ``_sse_read_lines``; close with ``resp.close()``)."""
|
||||||
conn = HTTPConnection("127.0.0.1", port, timeout=30)
|
conn = HTTPConnection("127.0.0.1", port, timeout=30)
|
||||||
@@ -355,8 +362,12 @@ async def test_post_wrong_content_type_400(gw):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_post_oversize_body_413(gw):
|
async def test_post_oversize_body_413(gw):
|
||||||
big = json.dumps(_frame_json({"type": "ping", "payload": {"pad": "x" * (1024 * 1024 + 1)}}))
|
big = json.dumps(
|
||||||
status, _ = await asyncio.to_thread(_request, http_port(gw), "POST", "/v1/frame", body=big)
|
_frame_json({"type": "ping", "payload": {"pad": "x" * (1024 * 1024 + 1)}})
|
||||||
|
)
|
||||||
|
status, _ = await asyncio.to_thread(
|
||||||
|
_request, http_port(gw), "POST", "/v1/frame", body=big
|
||||||
|
)
|
||||||
assert status == 413
|
assert status == 413
|
||||||
|
|
||||||
|
|
||||||
@@ -367,7 +378,12 @@ async def test_post_empty_message_400(gw):
|
|||||||
_post_frame,
|
_post_frame,
|
||||||
http_port(gw),
|
http_port(gw),
|
||||||
_frame_json(
|
_frame_json(
|
||||||
{"id": 7, "type": "message.send", "chat_id": CHAT_ID, "payload": {"text": " "}}
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "message.send",
|
||||||
|
"chat_id": CHAT_ID,
|
||||||
|
"payload": {"text": " "},
|
||||||
|
}
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
assert status == 400
|
assert status == 400
|
||||||
@@ -666,7 +682,9 @@ def _upload(
|
|||||||
"X-Iris-Media-Ref": media_ref,
|
"X-Iris-Media-Ref": media_ref,
|
||||||
"X-Iris-Media-Kind": kind,
|
"X-Iris-Media-Kind": kind,
|
||||||
"X-Iris-Media-Filename": filename,
|
"X-Iris-Media-Filename": filename,
|
||||||
"X-Iris-Media-Sha256": sha256 if sha256 is not None else hashlib.sha256(data).hexdigest(),
|
"X-Iris-Media-Sha256": sha256
|
||||||
|
if sha256 is not None
|
||||||
|
else hashlib.sha256(data).hexdigest(),
|
||||||
}
|
}
|
||||||
status, payload = _request(
|
status, payload = _request(
|
||||||
port,
|
port,
|
||||||
@@ -772,7 +790,9 @@ async def test_media_pull_ok(gw):
|
|||||||
str(img), "image", "image/png", "http_pull_test.png", len(PNG_1X1)
|
str(img), "image", "image/png", "http_pull_test.png", len(PNG_1X1)
|
||||||
)
|
)
|
||||||
port = http_port(gw)
|
port = http_port(gw)
|
||||||
status, payload = await asyncio.to_thread(_request, port, "GET", f"/v1/media/{entry.media_id}")
|
status, payload = await asyncio.to_thread(
|
||||||
|
_request, port, "GET", f"/v1/media/{entry.media_id}"
|
||||||
|
)
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert payload == PNG_1X1
|
assert payload == PNG_1X1
|
||||||
conn = HTTPConnection("127.0.0.1", port, timeout=10)
|
conn = HTTPConnection("127.0.0.1", port, timeout=10)
|
||||||
@@ -791,7 +811,9 @@ async def test_media_pull_ok(gw):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_media_pull_unknown_404(gw):
|
async def test_media_pull_unknown_404(gw):
|
||||||
port = http_port(gw)
|
port = http_port(gw)
|
||||||
status, payload = await asyncio.to_thread(_request, port, "GET", "/v1/media/md_nope")
|
status, payload = await asyncio.to_thread(
|
||||||
|
_request, port, "GET", "/v1/media/md_nope"
|
||||||
|
)
|
||||||
body = json.loads(payload)
|
body = json.loads(payload)
|
||||||
assert status == 404
|
assert status == 404
|
||||||
assert body["payload"]["code"] == "not_found"
|
assert body["payload"]["code"] == "not_found"
|
||||||
@@ -801,9 +823,13 @@ async def test_media_pull_unknown_404(gw):
|
|||||||
async def test_media_pull_denied_path_404(gw):
|
async def test_media_pull_denied_path_404(gw):
|
||||||
"""Known id, but the path fails delivery validation (denylist) — same
|
"""Known id, but the path fails delivery validation (denylist) — same
|
||||||
re-check at pull time as the WS path."""
|
re-check at pull time as the WS path."""
|
||||||
entry = gw._media.register_outbound("/etc/passwd", "document", "text/plain", "passwd", 100)
|
entry = gw._media.register_outbound(
|
||||||
|
"/etc/passwd", "document", "text/plain", "passwd", 100
|
||||||
|
)
|
||||||
port = http_port(gw)
|
port = http_port(gw)
|
||||||
status, payload = await asyncio.to_thread(_request, port, "GET", f"/v1/media/{entry.media_id}")
|
status, payload = await asyncio.to_thread(
|
||||||
|
_request, port, "GET", f"/v1/media/{entry.media_id}"
|
||||||
|
)
|
||||||
body = json.loads(payload)
|
body = json.loads(payload)
|
||||||
assert status == 404
|
assert status == 404
|
||||||
assert body["payload"]["code"] == "not_found"
|
assert body["payload"]["code"] == "not_found"
|
||||||
@@ -8,7 +8,7 @@ while building the Kotlin client.
|
|||||||
Usage::
|
Usage::
|
||||||
|
|
||||||
hermes gateway & # with the iris plugin
|
hermes gateway & # with the iris plugin
|
||||||
python gateway-plugin/tests/ws_probe.py --token <IRIS_TOKEN> \
|
python tests/ws_probe.py --token <IRIS_TOKEN> \
|
||||||
--send "hello"
|
--send "hello"
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
@@ -136,9 +136,7 @@ def _print_frame(raw):
|
|||||||
f"preview={str(payload.get('preview'))[:80]!r}"
|
f"preview={str(payload.get('preview'))[:80]!r}"
|
||||||
)
|
)
|
||||||
elif ftype == "tool.progress":
|
elif ftype == "tool.progress":
|
||||||
extra = (
|
extra = f" idx={payload.get('index')} name={payload.get('name')!r} note={payload.get('note')!r}"
|
||||||
f" idx={payload.get('index')} name={payload.get('name')!r} note={payload.get('note')!r}"
|
|
||||||
)
|
|
||||||
elif ftype == "tool.end":
|
elif ftype == "tool.end":
|
||||||
extra = (
|
extra = (
|
||||||
f" idx={payload.get('index')} name={payload.get('name')!r} "
|
f" idx={payload.get('index')} name={payload.get('name')!r} "
|
||||||
@@ -147,7 +145,9 @@ def _print_frame(raw):
|
|||||||
elif ftype == "commentary":
|
elif ftype == "commentary":
|
||||||
extra = f" id={payload.get('message_id')} text={(payload.get('text') or '')[:120]!r}"
|
extra = f" id={payload.get('message_id')} text={(payload.get('text') or '')[:120]!r}"
|
||||||
elif ftype == "hello.ack":
|
elif ftype == "hello.ack":
|
||||||
extra = f" caps={payload.get('server_caps')} cursor={payload.get('sync_cursor')}"
|
extra = (
|
||||||
|
f" caps={payload.get('server_caps')} cursor={payload.get('sync_cursor')}"
|
||||||
|
)
|
||||||
elif ftype == "error":
|
elif ftype == "error":
|
||||||
extra = f" code={payload.get('code')} msg={payload.get('message')!r}"
|
extra = f" code={payload.get('code')} msg={payload.get('message')!r}"
|
||||||
elif ftype == "typing":
|
elif ftype == "typing":
|
||||||
@@ -254,34 +254,54 @@ def _evaluate_assertions(args, st: _TurnState) -> list[tuple[int, bool, str]]:
|
|||||||
if "message.start" in events and "message.stop" in events:
|
if "message.start" in events and "message.stop" in events:
|
||||||
i_start = events.index("message.start")
|
i_start = events.index("message.start")
|
||||||
i_stop = events.index("message.stop")
|
i_stop = events.index("message.stop")
|
||||||
if any(i_start < i < i_stop for i, e in enumerate(events) if e == "message.update"):
|
if any(
|
||||||
|
i_start < i < i_stop
|
||||||
|
for i, e in enumerate(events)
|
||||||
|
if e == "message.update"
|
||||||
|
):
|
||||||
ok = True
|
ok = True
|
||||||
break
|
break
|
||||||
results.append(
|
results.append(
|
||||||
(10, ok, "assert-turn: no message.start -> >=1 message.update -> message.stop")
|
(
|
||||||
|
10,
|
||||||
|
ok,
|
||||||
|
"assert-turn: no message.start -> >=1 message.update -> message.stop",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if args.assert_reasoning:
|
if args.assert_reasoning:
|
||||||
reasoning = st.final_stop_reasoning or st.final_message_reasoning
|
reasoning = st.final_stop_reasoning or st.final_message_reasoning
|
||||||
results.append(
|
results.append(
|
||||||
(11, bool(reasoning), "assert-reasoning: final message has no non-empty reasoning")
|
(
|
||||||
|
11,
|
||||||
|
bool(reasoning),
|
||||||
|
"assert-reasoning: final message has no non-empty reasoning",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if args.assert_tools:
|
if args.assert_tools:
|
||||||
ok = bool(st.tool_starts) and bool(st.tool_starts & st.tool_ends)
|
ok = bool(st.tool_starts) and bool(st.tool_starts & st.tool_ends)
|
||||||
results.append((12, ok, "assert-tools: no tool.start with a matching tool.end"))
|
results.append((12, ok, "assert-tools: no tool.start with a matching tool.end"))
|
||||||
if args.assert_commentary:
|
if args.assert_commentary:
|
||||||
results.append((13, st.commentary >= 1, "assert-commentary: no commentary frame"))
|
results.append(
|
||||||
|
(13, st.commentary >= 1, "assert-commentary: no commentary frame")
|
||||||
|
)
|
||||||
if args.assert_read_receipt:
|
if args.assert_read_receipt:
|
||||||
if st.read_receipt is None:
|
if st.read_receipt is None:
|
||||||
print("== SKIP: no read.receipt frame (M7 frame not live on this gateway)")
|
print("== SKIP: no read.receipt frame (M7 frame not live on this gateway)")
|
||||||
elif not st.read_receipt:
|
elif not st.read_receipt:
|
||||||
results.append(
|
results.append(
|
||||||
(18, False, "assert-read-receipt: read.receipt arrived before the sent message")
|
(
|
||||||
|
18,
|
||||||
|
False,
|
||||||
|
"assert-read-receipt: read.receipt arrived before the sent message",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if args.assert_status:
|
if args.assert_status:
|
||||||
if not st.status_seen:
|
if not st.status_seen:
|
||||||
print("== SKIP: no status frame (M7 frame not live on this gateway)")
|
print("== SKIP: no status frame (M7 frame not live on this gateway)")
|
||||||
elif st.status_empty:
|
elif st.status_empty:
|
||||||
results.append((19, False, "assert-status: status frame arrived with an empty payload"))
|
results.append(
|
||||||
|
(19, False, "assert-status: status frame arrived with an empty payload")
|
||||||
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@@ -391,7 +411,12 @@ def run_http(args, base: str) -> int:
|
|||||||
host,
|
host,
|
||||||
port,
|
port,
|
||||||
headers,
|
headers,
|
||||||
{"v": 1, "id": 1, "type": "channel.create", "payload": {"name": args.channel_create}},
|
{
|
||||||
|
"v": 1,
|
||||||
|
"id": 1,
|
||||||
|
"type": "channel.create",
|
||||||
|
"payload": {"name": args.channel_create},
|
||||||
|
},
|
||||||
"channel.created",
|
"channel.created",
|
||||||
30,
|
30,
|
||||||
)
|
)
|
||||||
@@ -477,7 +502,12 @@ def run_http(args, base: str) -> int:
|
|||||||
host,
|
host,
|
||||||
port,
|
port,
|
||||||
headers,
|
headers,
|
||||||
{"v": 1, "id": 1, "type": "fcm.register", "payload": {"token": args.fcm_token}},
|
{
|
||||||
|
"v": 1,
|
||||||
|
"id": 1,
|
||||||
|
"type": "fcm.register",
|
||||||
|
"payload": {"token": args.fcm_token},
|
||||||
|
},
|
||||||
"fcm.registered",
|
"fcm.registered",
|
||||||
30,
|
30,
|
||||||
)
|
)
|
||||||
@@ -513,7 +543,10 @@ def run_http(args, base: str) -> int:
|
|||||||
if data is not None and data.get("chat_id") == args.watch:
|
if data is not None and data.get("chat_id") == args.watch:
|
||||||
ftype = data.get("type")
|
ftype = data.get("type")
|
||||||
payload = data.get("payload") or {}
|
payload = data.get("payload") or {}
|
||||||
if ftype == "message" and payload.get("role") in ("assistant", "cron"):
|
if ftype == "message" and payload.get("role") in (
|
||||||
|
"assistant",
|
||||||
|
"cron",
|
||||||
|
):
|
||||||
print(
|
print(
|
||||||
f"== message landed in {args.watch}: {str(payload.get('text'))[:120]!r}"
|
f"== message landed in {args.watch}: {str(payload.get('text'))[:120]!r}"
|
||||||
)
|
)
|
||||||
@@ -628,7 +661,11 @@ def run_http(args, base: str) -> int:
|
|||||||
got_final = True
|
got_final = True
|
||||||
if ftype == "message.stop":
|
if ftype == "message.stop":
|
||||||
seen_final_frame = True
|
seen_final_frame = True
|
||||||
if ftype == "typing" and payload.get("on") is False and seen_final_frame:
|
if (
|
||||||
|
ftype == "typing"
|
||||||
|
and payload.get("on") is False
|
||||||
|
and seen_final_frame
|
||||||
|
):
|
||||||
got_final = True
|
got_final = True
|
||||||
cur_data = []
|
cur_data = []
|
||||||
return got_final
|
return got_final
|
||||||
@@ -673,7 +710,9 @@ def main() -> int:
|
|||||||
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
|
p.add_argument("--device", default=f"probe-{uuid.uuid4().hex[:8]}")
|
||||||
p.add_argument("--send", default="hello")
|
p.add_argument("--send", default="hello")
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--upload", default="", help="M4: file to upload (chunked) and attach via media_refs"
|
"--upload",
|
||||||
|
default="",
|
||||||
|
help="M4: file to upload (chunked) and attach via media_refs",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--pull-offer",
|
"--pull-offer",
|
||||||
@@ -686,14 +725,18 @@ def main() -> int:
|
|||||||
default=None,
|
default=None,
|
||||||
help="M5: send sync {cursor} after pairing, print replay, exit",
|
help="M5: send sync {cursor} after pairing, print replay, exit",
|
||||||
)
|
)
|
||||||
p.add_argument("--fcm-token", default="", help="M5: FCM token to attach to the hello payload")
|
p.add_argument(
|
||||||
|
"--fcm-token", default="", help="M5: FCM token to attach to the hello payload"
|
||||||
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--fcm-reg",
|
"--fcm-reg",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="M5: send fcm.register after pairing (uses --fcm-token)",
|
help="M5: send fcm.register after pairing (uses --fcm-token)",
|
||||||
)
|
)
|
||||||
p.add_argument("--timeout", type=float, default=120.0)
|
p.add_argument("--timeout", type=float, default=120.0)
|
||||||
p.add_argument("--authfail", action="store_true", help="expect an auth rejection (wrong token)")
|
p.add_argument(
|
||||||
|
"--authfail", action="store_true", help="expect an auth rejection (wrong token)"
|
||||||
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--assert-turn",
|
"--assert-turn",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -705,9 +748,13 @@ def main() -> int:
|
|||||||
help="assert the final message.stop carries non-empty reasoning",
|
help="assert the final message.stop carries non-empty reasoning",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--assert-tools", action="store_true", help="assert >=1 tool.start with a matching tool.end"
|
"--assert-tools",
|
||||||
|
action="store_true",
|
||||||
|
help="assert >=1 tool.start with a matching tool.end",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--assert-commentary", action="store_true", help="assert >=1 commentary frame"
|
||||||
)
|
)
|
||||||
p.add_argument("--assert-commentary", action="store_true", help="assert >=1 commentary frame")
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--assert-read-receipt",
|
"--assert-read-receipt",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -719,10 +766,15 @@ def main() -> int:
|
|||||||
help="assert a status frame is received (SKIP if absent; M7)",
|
help="assert a status frame is received (SKIP if absent; M7)",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--search", default="", help="M3: send search {query, scope, limit}, assert >=1 hit"
|
"--search",
|
||||||
|
default="",
|
||||||
|
help="M3: send search {query, scope, limit}, assert >=1 hit",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--scope", choices=("all", "chat"), default="all", help="search scope (default all)"
|
"--scope",
|
||||||
|
choices=("all", "chat"),
|
||||||
|
default="all",
|
||||||
|
help="search scope (default all)",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--chat-id",
|
"--chat-id",
|
||||||
@@ -730,12 +782,20 @@ def main() -> int:
|
|||||||
help="chat_id for --scope chat (default default)",
|
help="chat_id for --scope chat (default default)",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--channel-create", default="", help="M3: create a channel, print its chat_id, exit"
|
"--channel-create",
|
||||||
|
default="",
|
||||||
|
help="M3: create a channel, print its chat_id, exit",
|
||||||
)
|
)
|
||||||
p.add_argument("--channel-delete", default="", help="M3: delete (archive) a channel, exit")
|
|
||||||
p.add_argument("--channel-list", action="store_true", help="M3: list channels, exit")
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--watch", default="", help="wait up to --timeout for a message to land in this chat_id"
|
"--channel-delete", default="", help="M3: delete (archive) a channel, exit"
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--channel-list", action="store_true", help="M3: list channels, exit"
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--watch",
|
||||||
|
default="",
|
||||||
|
help="wait up to --timeout for a message to land in this chat_id",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--offer-grace",
|
"--offer-grace",
|
||||||
@@ -766,7 +826,9 @@ def main() -> int:
|
|||||||
if not args.token and not args.authfail:
|
if not args.token and not args.authfail:
|
||||||
p.error("--token (or $IRIS_TOKEN) is required")
|
p.error("--token (or $IRIS_TOKEN) is required")
|
||||||
if args.assert_read_receipt and not args.send:
|
if args.assert_read_receipt and not args.send:
|
||||||
p.error("--assert-read-receipt requires --send (the receipt must follow the sent message)")
|
p.error(
|
||||||
|
"--assert-read-receipt requires --send (the receipt must follow the sent message)"
|
||||||
|
)
|
||||||
# HTTP is the only transport (docs/19): derive the http(s) base from the
|
# HTTP is the only transport (docs/19): derive the http(s) base from the
|
||||||
# --url (legacy ws(s)://host:8790/ws -> http(s)://host:8791) unless
|
# --url (legacy ws(s)://host:8790/ws -> http(s)://host:8791) unless
|
||||||
# --http-url is given.
|
# --http-url is given.
|
||||||
Reference in new issue
Block a user