From 2c20b1c8a53cb5e920f66a73e765429b8ac95a39 Mon Sep 17 00:00:00 2001 From: ARIA Date: Wed, 23 Sep 2026 15:10:16 +0200 Subject: [PATCH] fix(http): don't let a half-open TLS connection wedge the accept loop A client that completes TCP but vanishes mid-TLS-handshake (e.g. a phone losing its network/VPN while traveling) blocked ssl.SSLSocket.accept() inside serve_forever forever: the gateway stopped accepting any new device connections (the app could not reconnect), and on the next restart httpd.shutdown() froze the whole event loop until the shutdown watchdog killed the process (ARIA journal 2026-09-11 / 2026-09-23). - Move the TLS handshake out of the accept loop: it now runs in the per-connection thread under a hard timeout (HANDSHAKE_TIMEOUT_S, 10 s); a failed/timed-out handshake just closes the socket. - stop() no longer blocks the event loop: shutdown()/server_close()/ join run in an executor under asyncio.wait_for(10 s); if the bound expires the daemon threads are abandoned. - Regression test: a silent half-open TCP connection must not stop fresh TLS connections from being served, and stop() must stay bounded. --- gateway-plugin/http_server.py | 94 +++++++++++++++++++++--- tests/test_android_http.py | 134 +++++++++++++++++++++++++++++++++- 2 files changed, 216 insertions(+), 12 deletions(-) diff --git a/gateway-plugin/http_server.py b/gateway-plugin/http_server.py index 5889e5d..d84f92d 100644 --- a/gateway-plugin/http_server.py +++ b/gateway-plugin/http_server.py @@ -45,6 +45,7 @@ import contextlib import json import logging import queue +import socket import ssl import threading import time @@ -220,7 +221,14 @@ class HttpServer: if self._adapter.http_cert and self._adapter.http_key: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_cert_chain(self._adapter.http_cert, self._adapter.http_key) - httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True) + # The handshake runs in the per-connection thread with a + # hard timeout (see _ThreadingHTTPD.process_request). + # Wrapping the *listening* socket here instead would make + # serve_forever's accept() block inside do_handshake() on + # a half-open connection (TCP established, client gone + # mid-handshake), wedging ALL new device connections until + # the gateway is restarted. + httpd.set_tls(ctx) except Exception as e: logger.warning("iris: HTTP server disabled (bind %s:%s failed: %s)", host, port, e) self._release_lock() @@ -245,17 +253,35 @@ class HttpServer: s.q.put_nowait(_STOP) httpd = self._httpd self._httpd = None - if httpd is not None: - # shutdown() must be called from a thread other than the one - # running serve_forever(); we are on the asyncio loop thread. - with contextlib.suppress(Exception): - httpd.shutdown() - with contextlib.suppress(Exception): - httpd.server_close() t = self._thread self._thread = None - if t is not None and t is not threading.current_thread(): - t.join(timeout=5.0) + if httpd is not None or t is not None: + # shutdown() blocks until the serve_forever loop exits and + # server_close() may join handler threads — both must run on a + # worker thread (never the asyncio loop thread) with a hard + # timeout, or a wedged server would freeze the whole gateway. + # The threads are daemons: if the bounded wait expires they die + # with the process and there is nothing left to do. + loop = asyncio.get_running_loop() + + def _stop_httpd() -> None: + if httpd is not None: + with contextlib.suppress(Exception): + httpd.shutdown() + with contextlib.suppress(Exception): + httpd.server_close() + if t is not None and t is not threading.current_thread(): + t.join(timeout=5.0) + + try: + await asyncio.wait_for( + loop.run_in_executor(None, _stop_httpd), + timeout=10.0, + ) + except Exception: + logger.warning( + "iris: HTTP server teardown did not finish in time; abandoning daemon threads" + ) self._release_lock() def _release_lock(self) -> None: @@ -783,14 +809,60 @@ class HttpServer: class _ThreadingHTTPD(ThreadingHTTPServer): """One thread per connection (fine at single-user scale); daemon - threads so a stuck handler can't block process exit.""" + threads so a stuck handler can't block process exit. + + With TLS enabled (``set_tls``) the handshake runs in the + per-connection thread under a hard timeout — never in the + ``serve_forever`` accept loop. ``ssl.SSLSocket.accept()`` would + otherwise block that loop inside ``do_handshake()`` on a half-open + connection (TCP established but the client vanished mid-handshake, + e.g. a phone losing its network/VPN), and the gateway would stop + accepting any new device connections until it is restarted. + """ daemon_threads = True allow_reuse_address = True + # A client that completes TCP but never finishes the TLS handshake + # must not hold the connection open indefinitely. + HANDSHAKE_TIMEOUT_S = 10.0 + def __init__(self, addr: tuple[str, int], http_server: HttpServer): super().__init__(addr, _Handler) self.http_server = http_server + self._tls_ctx: ssl.SSLContext | None = None + + def set_tls(self, ctx: ssl.SSLContext) -> None: + self._tls_ctx = ctx + + def process_request( # noqa: A003 # type: ignore[override] + self, request: socket.socket, client_address: Any + ) -> None: + """Spawn the handler thread; with TLS, the handshake happens in + that thread first, under ``HANDSHAKE_TIMEOUT_S`` (see class + docstring). A failed/timed-out handshake just closes the socket — + the accept loop is never blocked by it.""" + if self._tls_ctx is None: + super().process_request(request, client_address) + return + tls_ctx = self._tls_ctx + + def _handshake_then_handle() -> None: + try: + request.settimeout(self.HANDSHAKE_TIMEOUT_S) + # wrap_socket() performs the handshake (default + # do_handshake_on_connect=True); restore blocking mode for + # the request handler afterwards. + tls_sock = tls_ctx.wrap_socket(request, server_side=True) + tls_sock.settimeout(None) + except OSError as e: # ssl.SSLError, timeout, reset, ... + with contextlib.suppress(OSError): + request.close() + logger.debug("iris http: TLS handshake failed (%s): %s", client_address, e) + return + super(_ThreadingHTTPD, self).process_request(tls_sock, client_address) + + threading.Thread(target=_handshake_then_handle, name="iris-tls", daemon=True).start() class _Handler(BaseHTTPRequestHandler): diff --git a/tests/test_android_http.py b/tests/test_android_http.py index 781164f..5ab798b 100644 --- a/tests/test_android_http.py +++ b/tests/test_android_http.py @@ -29,11 +29,14 @@ import asyncio import base64 import contextlib import importlib.util +import ipaddress import json import os +import socket +import ssl import sys import time -from http.client import HTTPConnection +from http.client import HTTPConnection, HTTPSConnection from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -835,6 +838,135 @@ async def test_media_pull_denied_path_404(gw): assert body["payload"]["code"] == "not_found" +# ── TLS: a half-open connection must not wedge the accept loop ───────────── +# +# Regression (ARIA journal 2026-09-11 / 2026-09-23): the listening socket +# used to be wrapped in a server-side ssl.SSLSocket, so serve_forever's +# accept() ran the TLS handshake inline. A client that completed TCP but +# vanished mid-handshake (a phone losing its network/VPN while traveling) +# blocked do_handshake() forever: the gateway stopped accepting ANY new +# device connections (the app could not reconnect), and on the next restart +# httpd.shutdown() froze the whole event loop until the shutdown watchdog +# killed the process. + + +def _make_self_signed_cert(tmp_path: Path) -> tuple[Path, Path] | None: + """Self-signed cert + key for the TLS tests; None when + ``cryptography`` is unavailable (the tests then skip).""" + try: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + except ImportError: + return None + import datetime + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "iris-test")]) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension( + x509.SubjectAlternativeName( + [ + x509.DNSName("localhost"), + x509.IPAddress(ipaddress.ip_address("127.0.0.1")), + ] + ), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_path = tmp_path / "iris-test.crt" + key_path = tmp_path / "iris-test.key" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_path, key_path + + +@pytest_asyncio.fixture +async def gw_tls(adapter, tmp_path, monkeypatch): + """Connected adapter with the HTTP leg TLS-enabled; the handshake + timeout is shortened so the half-open connection cleans itself up + quickly.""" + paths = _make_self_signed_cert(tmp_path) + if paths is None: + pytest.skip("cryptography not available; TLS wedge test skipped") + cert_path, key_path = paths + plugin = _load_plugin() + monkeypatch.setattr( + plugin.http_server._ThreadingHTTPD, "HANDSHAKE_TIMEOUT_S", 0.5, raising=False + ) + adapter.http_cert = str(cert_path) + adapter.http_key = str(key_path) + await adapter.connect() + try: + yield adapter + finally: + await adapter.disconnect() + + +def _tls_health(port: int) -> int: + """GET /v1/health over a fresh TLS connection; returns the status.""" + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + conn = HTTPSConnection("127.0.0.1", port, timeout=5.0, context=ctx) + conn.request("GET", "/v1/health") + resp = conn.getresponse() + status = resp.status + resp.read() + conn.close() + return status + + +@pytest.mark.asyncio +async def test_half_open_tls_connection_does_not_wedge_accept_loop(gw_tls): + """A client that completes TCP but never finishes the TLS handshake + must not stop the server from accepting new connections (see section + comment for the incident).""" + port = http_port(gw_tls) + + # 1) Half-open connection: TCP established, then silence — the + # phone-loses-its-VPN scenario (the ClientHello never arrives). + wedge = socket.create_connection(("127.0.0.1", port), timeout=5.0) + try: + # 2) While the half-open connection sits un-handshaked, a fresh, + # well-formed TLS connection must still be accepted promptly. + deadline = time.monotonic() + 10.0 + status = None + while time.monotonic() < deadline: + try: + status = await asyncio.to_thread(_tls_health, port) + break + except OSError: + await asyncio.sleep(0.2) + assert status == 200, f"health over TLS failed (status={status})" + + # 3) Teardown must stay bounded with the half-open connection still + # open: stop() used to block the event loop on httpd.shutdown() + # until the shutdown watchdog killed the process. + t0 = time.monotonic() + await gw_tls._http_server.stop() + assert time.monotonic() - t0 < 15.0 + finally: + with contextlib.suppress(OSError): + wedge.close() + + # ── Helpers ───────────────────────────────────────────────────────────────── -- 2.54.0