M3: channels/threads + cron delivery + search + outbox sync
Gateway plugin: - ChannelDirectory (SQLite): channels + threads, friendly-name resolution - FTS5 search bridge over state.db (scope all/chat, LIKE fallback) - Outbox (monotonic cursor, 72h retention) for reconnect catch-up - adapter: channel.* handlers, search, sync, cron target parsing - ws_server: M3 frame routing + hello.ack sync cursor App (KMP): - ChannelStore (directory cache) + lane-aware ChatStore (per chat/thread) - GatewayClient.sendFrame; IrisController channel/search/nav actions - ChatScreen: channel drawer, thread toggle, topic switcher, search overlay
This commit is contained in:
ARIA committed
1 parent
218c50d688
commit
9b511fdd28
13 files changed
+2035
-259
No files matched your search
+233
-5
@@ -1,8 +1,236 @@
|
||||
"""FTS5 session search bridge.
|
||||
"""FTS5 session search bridge (M3).
|
||||
|
||||
Bridges the ``search {query, scope, chat_id?, thread_id?}`` frame to the
|
||||
hermes session store (SQLite + FTS5, ``hermes_state_search.py``) and returns
|
||||
``search.results``. Scope: "all" (everywhere) or "chat" (this chat/channel).
|
||||
Bridges the ``search {query, scope, chat_id?, thread_id?}`` frame to the hermes
|
||||
session store (SQLite + FTS5, ``hermes_state.py`` / ``hermes_state_search.py``)
|
||||
and returns ``search.results`` hits.
|
||||
|
||||
The session DB (``get_hermes_home()/"state.db"``) is opened **read-only** --
|
||||
search never writes to the store. Two query paths:
|
||||
|
||||
* **FTS5** (primary): ``messages_fts MATCH <sanitized>`` with BM25 ranking.
|
||||
* **LIKE** (fallback): when the FTS5 table is absent (FTS disabled / fresh DB)
|
||||
or the MATCH raises, a substring scan over ``messages.content``.
|
||||
|
||||
Scope:
|
||||
* ``"all"`` -- every channel/thread/session.
|
||||
* ``"chat"`` -- restrict to the given ``chat_id`` (and optional ``thread_id``).
|
||||
|
||||
Each hit: ``{message_id, chat_id, thread_id, role, snippet, ts}`` where
|
||||
``message_id`` is the session-store row id (string) and ``ts`` is epoch
|
||||
milliseconds. The app navigates to the hit's channel/thread and matches the
|
||||
message by timestamp to scroll + highlight.
|
||||
|
||||
Privacy: search is local to the user's own hermes home; no data leaves the
|
||||
machine.
|
||||
|
||||
Milestone M3.
|
||||
"""
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_QUERY_CHARS = 200
|
||||
DEFAULT_LIMIT = 20
|
||||
MAX_LIMIT = 100
|
||||
|
||||
# FTS5 special chars (mirror of hermes_state_search._FTS5_SPECIAL_CHARS) for the
|
||||
# fallback sanitizer when the real one can't be imported.
|
||||
_FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\''
|
||||
_FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]")
|
||||
|
||||
|
||||
def _sanitize(query: str) -> str:
|
||||
"""Sanitize user input for a safe FTS5 MATCH.
|
||||
|
||||
Prefers the gateway's own sanitizer (exact parity with session_search);
|
||||
falls back to a simplified strip-and-quote pass when it can't be imported.
|
||||
"""
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return ""
|
||||
try:
|
||||
from hermes_state_search import SessionSearchMixin
|
||||
|
||||
return SessionSearchMixin._sanitize_fts5_query(q)
|
||||
except Exception:
|
||||
return _sanitize_fallback(q)
|
||||
|
||||
|
||||
def _sanitize_fallback(query: str) -> str:
|
||||
q = query[:MAX_QUERY_CHARS]
|
||||
q = _FTS5_SPECIAL_RE.sub(" ", q)
|
||||
if "%" in q:
|
||||
q = q.replace("%", " ")
|
||||
q = re.sub(r"\*+", "*", q)
|
||||
q = re.sub(r"(^|\s)\*", r"\1", q)
|
||||
q = re.sub(r"(?i)^(AND|OR|NOT)\b\s*", "", q.strip())
|
||||
q = re.sub(r"(?i)\s+(AND|OR|NOT)\s*$", "", q.strip())
|
||||
q = re.sub(r"\b(\w+(?:[._-]\w+)+)\b", r'"\1"', q)
|
||||
return q.strip()
|
||||
|
||||
|
||||
def _fts_available(conn: sqlite3.Connection) -> bool:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name = 'messages_fts' LIMIT 1"
|
||||
).fetchone()
|
||||
return row is not None
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
|
||||
|
||||
def _scope_clauses(
|
||||
scope: str, chat_id: Optional[str], thread_id: Optional[str]
|
||||
) -> Tuple[List[str], List[Any]]:
|
||||
"""Build the scope WHERE clauses + params (empty for scope='all')."""
|
||||
clauses: List[str] = []
|
||||
params: List[Any] = []
|
||||
if scope == "chat" and chat_id:
|
||||
clauses.append("s.chat_id = ?")
|
||||
params.append(chat_id)
|
||||
if thread_id:
|
||||
clauses.append("s.thread_id = ?")
|
||||
params.append(thread_id)
|
||||
return clauses, params
|
||||
|
||||
|
||||
def _row_to_hit(row: sqlite3.Row) -> Dict[str, Any]:
|
||||
ts = row["timestamp"]
|
||||
try:
|
||||
ts_ms = int(float(ts) * 1000)
|
||||
except (TypeError, ValueError):
|
||||
ts_ms = 0
|
||||
return {
|
||||
"message_id": str(row["id"]),
|
||||
"chat_id": row["chat_id"],
|
||||
"thread_id": row["thread_id"],
|
||||
"role": row["role"],
|
||||
"snippet": row["snippet"] or "",
|
||||
"ts": ts_ms,
|
||||
}
|
||||
|
||||
|
||||
def _fts_query(
|
||||
conn: sqlite3.Connection,
|
||||
query: str,
|
||||
scope: str,
|
||||
chat_id: Optional[str],
|
||||
thread_id: Optional[str],
|
||||
limit: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
where = ["messages_fts MATCH ?", "(m.active = 1 OR m.compacted = 1)"]
|
||||
params: List[Any] = [query]
|
||||
scope_clauses, scope_params = _scope_clauses(scope, chat_id, thread_id)
|
||||
where.extend(scope_clauses)
|
||||
params.extend(scope_params)
|
||||
params.extend([limit])
|
||||
sql = f"""
|
||||
SELECT
|
||||
m.id,
|
||||
m.role,
|
||||
snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet,
|
||||
m.timestamp,
|
||||
s.chat_id,
|
||||
s.thread_id
|
||||
FROM messages_fts
|
||||
JOIN messages m ON m.id = messages_fts.rowid
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE {' AND '.join(where)}
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
"""
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [_row_to_hit(r) for r in rows]
|
||||
|
||||
|
||||
def _like_query(
|
||||
conn: sqlite3.Connection,
|
||||
query: str,
|
||||
scope: str,
|
||||
chat_id: Optional[str],
|
||||
thread_id: Optional[str],
|
||||
limit: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Substring fallback when FTS5 is unavailable."""
|
||||
# First plain word of the query is the LIKE needle (best-effort).
|
||||
needle = re.split(r"\s+", query.strip(), maxsplit=1)[0].strip('"')
|
||||
if not needle:
|
||||
return []
|
||||
like = f"%{needle}%"
|
||||
where = ["(m.active = 1 OR m.compacted = 1)", "m.content LIKE ?"]
|
||||
params: List[Any] = [like]
|
||||
scope_clauses, scope_params = _scope_clauses(scope, chat_id, thread_id)
|
||||
where.extend(scope_clauses)
|
||||
params.extend(scope_params)
|
||||
params.extend([limit])
|
||||
sql = f"""
|
||||
SELECT
|
||||
m.id,
|
||||
m.role,
|
||||
substr(m.content, max(1, instr(m.content, ?) - 40), 120) AS snippet,
|
||||
m.timestamp,
|
||||
s.chat_id,
|
||||
s.thread_id
|
||||
FROM messages m
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE {' AND '.join(where)}
|
||||
ORDER BY m.timestamp DESC
|
||||
LIMIT ?
|
||||
"""
|
||||
# The needle appears twice (LIKE + instr); params order: like, scope..., needle, limit
|
||||
full_params = [like, *scope_params, needle, limit]
|
||||
rows = conn.execute(sql, full_params).fetchall()
|
||||
return [_row_to_hit(r) for r in rows]
|
||||
|
||||
|
||||
def search(
|
||||
db_path: Path,
|
||||
query: str,
|
||||
scope: str = "all",
|
||||
chat_id: Optional[str] = None,
|
||||
thread_id: Optional[str] = None,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Run a scoped search over the session store. Returns a list of hits.
|
||||
|
||||
Never raises: any DB/FTS error yields an empty result (the caller sends an
|
||||
empty ``search.results``).
|
||||
"""
|
||||
sanitized = _sanitize(query)
|
||||
if not sanitized:
|
||||
return []
|
||||
db_path = Path(db_path)
|
||||
if not db_path.exists():
|
||||
return []
|
||||
limit = max(1, min(int(limit or DEFAULT_LIMIT), MAX_LIMIT))
|
||||
scope = (scope or "all").strip().lower()
|
||||
if scope not in ("all", "chat"):
|
||||
scope = "all"
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
logger.warning("android search: open failed: %s", e)
|
||||
return []
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
if _fts_available(conn):
|
||||
try:
|
||||
return _fts_query(conn, sanitized, scope, chat_id, thread_id, limit)
|
||||
except sqlite3.Error as e:
|
||||
logger.debug("android search: FTS5 failed, using LIKE: %s", e)
|
||||
return _like_query(conn, sanitized, scope, chat_id, thread_id, limit)
|
||||
except sqlite3.Error as e:
|
||||
logger.warning("android search: query failed: %s", e)
|
||||
return []
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
Reference in new issue
Block a user