diff --git a/tests/unit/test_settlement_after_delivery.py b/tests/unit/test_settlement_after_delivery.py new file mode 100644 index 0000000..f0e1e4d --- /dev/null +++ b/tests/unit/test_settlement_after_delivery.py @@ -0,0 +1,230 @@ +"""A settlement that fails after the response was served must not be free inference. + +These are the acceptance tests for the after-delivery settlement path. They +enter where production enters — ``_adjust_payment_for_tokens`` — against a real +database, and they assert the two things the old behaviour got wrong: a charge +whose commit was ambiguous is neither lost nor duplicated, and a request that +could not be charged is recorded as a write-off instead of looking free. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool +from sqlmodel import SQLModel +from sqlmodel.ext.asyncio.session import AsyncSession + +import routstr.auth as auth_module +from routstr.auth import ReservationSnapshot +from routstr.core.db import ApiKey, ReservationRelease +from routstr.payment.cost_calculation import MaxCostData + +KEY_HASH = "a" * 64 +RESERVATION_ID = "rel-1" +RESERVED_MSATS = 1_000 +BALANCE_MSATS = 10_000 +CHARGE_MSATS = 500 + + +@pytest.fixture +async def engine(tmp_path: Path) -> AsyncIterator[AsyncEngine]: + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'delivery.db'}", poolclass=NullPool + ) + async with engine.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + try: + yield engine + finally: + await engine.dispose() + + +async def _seed(engine: AsyncEngine) -> None: + async with AsyncSession(engine, expire_on_commit=False) as session: + session.add( + ApiKey( + hashed_key=KEY_HASH, + balance=BALANCE_MSATS, + reserved_balance=RESERVED_MSATS, + ) + ) + session.add( + ReservationRelease( + id=RESERVATION_ID, + key_hash=KEY_HASH, + billing_key_hash=KEY_HASH, + reserved_msats=RESERVED_MSATS, + status="active", + ) + ) + await session.commit() + + +def _bind_sessions(monkeypatch: pytest.MonkeyPatch, engine: AsyncEngine) -> None: + @asynccontextmanager + async def factory() -> AsyncIterator[AsyncSession]: + async with AsyncSession(engine, expire_on_commit=False) as session: + yield session + + monkeypatch.setattr(auth_module, "create_session", factory) + + +def _snapshot() -> ReservationSnapshot: + return ReservationSnapshot( + release_id=RESERVATION_ID, + key_hash=KEY_HASH, + billing_key_hash=KEY_HASH, + reserved_msats=RESERVED_MSATS, + ) + + +async def _state(engine: AsyncEngine) -> dict[str, Any]: + async with AsyncSession(engine, expire_on_commit=False) as session: + key = await session.get(ApiKey, KEY_HASH) + row = await session.get(ReservationRelease, RESERVATION_ID) + assert key is not None and row is not None + return { + "balance": key.balance, + "reserved_balance": key.reserved_balance, + "total_spent": key.total_spent, + "status": row.status, + } + + +async def _settle( + monkeypatch: pytest.MonkeyPatch, session: AsyncSession +) -> dict[str, Any]: + async def price(*_args: Any, **_kwargs: Any) -> Any: + return MaxCostData( + base_msats=CHARGE_MSATS, + input_msats=0, + output_msats=0, + total_msats=CHARGE_MSATS, + input_tokens=10, + output_tokens=5, + ) + + monkeypatch.setattr(auth_module, "calculate_cost", price) + key = await session.get(ApiKey, KEY_HASH) + assert key is not None + return await auth_module._adjust_payment_for_tokens( + key, + { + "model": "test-model", + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + }, + session, + RESERVED_MSATS, + None, + None, + _snapshot(), + ) + + +@pytest.mark.asyncio +async def test_a_commit_that_lands_then_raises_charges_the_request_once( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + """The exact ambiguity the old code released: the commit landed, the caller saw an error.""" + await _seed(engine) + _bind_sessions(monkeypatch, engine) + async with AsyncSession(engine, expire_on_commit=False) as session: + real_commit = session.commit + commits = {"count": 0} + + async def lands_then_raises() -> None: + commits["count"] += 1 + await real_commit() + if commits["count"] == 1: + raise RuntimeError("connection reset after commit") + + monkeypatch.setattr(session, "commit", lands_then_raises) + + cost = await _settle(monkeypatch, session) + + assert cost["charged_msats"] == CHARGE_MSATS + assert await _state(engine) == { + "balance": BALANCE_MSATS - CHARGE_MSATS, + "reserved_balance": 0, + "total_spent": CHARGE_MSATS, + "status": "charged", + } + + +@pytest.mark.asyncio +async def test_a_commit_that_never_landed_is_charged_by_a_fresh_session( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + await _seed(engine) + _bind_sessions(monkeypatch, engine) + async with AsyncSession(engine, expire_on_commit=False) as session: + + async def never_lands() -> None: + raise RuntimeError("database is locked") + + monkeypatch.setattr(session, "commit", never_lands) + + cost = await _settle(monkeypatch, session) + + assert cost["charged_msats"] == CHARGE_MSATS + assert await _state(engine) == { + "balance": BALANCE_MSATS - CHARGE_MSATS, + "reserved_balance": 0, + "total_spent": CHARGE_MSATS, + "status": "charged", + } + + +@pytest.mark.asyncio +async def test_an_unsettled_request_is_written_off_explicitly( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + """A served request nobody could charge is a loss, and the log has to say so.""" + await _seed(engine) + + def unavailable() -> Any: + raise RuntimeError("database is down") + + monkeypatch.setattr(auth_module, "create_session", unavailable) + monkeypatch.setattr(auth_module, "SETTLEMENT_RECOVERY_BACKOFF_SECONDS", 0.0) + + finalize_records: list[dict[str, Any]] = [] + + def capture(message: str, *args: Any, **kwargs: Any) -> None: + finalize_records.append(kwargs.get("extra", {})) + + monkeypatch.setattr(auth_module.payments_logger, "error", capture) + + async with AsyncSession(engine, expire_on_commit=False) as session: + real_commit = session.commit + calls = {"count": 0} + + async def fails_once() -> None: + calls["count"] += 1 + if calls["count"] == 1: + raise RuntimeError("database is locked") + await real_commit() + + monkeypatch.setattr(session, "commit", fails_once) + + cost = await _settle(monkeypatch, session) + + assert cost["charged_msats"] == 0 + state = await _state(engine) + assert state["balance"] == BALANCE_MSATS + assert state["status"] == "released" + + write_offs = [ + record + for record in finalize_records + if record.get("finalize_type") == "released_unsettled" + ] + assert write_offs, "an unsettled settlement must be recorded as a write-off" + assert write_offs[0]["cost_unsettled"] == CHARGE_MSATS + assert write_offs[0]["cost_charged"] == 0 diff --git a/tests/unit/test_settlement_recovery.py b/tests/unit/test_settlement_recovery.py new file mode 100644 index 0000000..02720bc --- /dev/null +++ b/tests/unit/test_settlement_recovery.py @@ -0,0 +1,180 @@ +"""A settlement that cannot commit must not become free inference. + +The charge is claimed with a conditional ``UPDATE ... WHERE status='active'``, +so the recovery path can retry in a fresh session without any risk of charging +twice: a second attempt either finds the row already ``charged`` — the first +commit landed — or still ``active``, in which case the first commit did not +land and the retry owns the charge. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool +from sqlmodel import SQLModel +from sqlmodel.ext.asyncio.session import AsyncSession + +import routstr.auth as auth_module +from routstr.auth import ReservationSnapshot, _recover_known_charge +from routstr.core.db import ApiKey, ReservationRelease + +KEY_HASH = "a" * 64 +RESERVATION_ID = "rel-1" +RESERVED_MSATS = 1_000 +BALANCE_MSATS = 10_000 +CHARGE_MSATS = 500 + + +@pytest.fixture +async def engine(tmp_path: Path) -> AsyncIterator[AsyncEngine]: + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'settlement.db'}", poolclass=NullPool + ) + async with engine.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + try: + yield engine + finally: + await engine.dispose() + + +async def _seed(engine: AsyncEngine, *, status: str = "active") -> None: + async with AsyncSession(engine, expire_on_commit=False) as session: + session.add( + ApiKey( + hashed_key=KEY_HASH, + balance=BALANCE_MSATS, + reserved_balance=RESERVED_MSATS, + ) + ) + session.add( + ReservationRelease( + id=RESERVATION_ID, + key_hash=KEY_HASH, + billing_key_hash=KEY_HASH, + reserved_msats=RESERVED_MSATS, + status=status, + ) + ) + await session.commit() + + +def _bind_sessions(monkeypatch: pytest.MonkeyPatch, engine: AsyncEngine) -> None: + @asynccontextmanager + async def factory() -> AsyncIterator[AsyncSession]: + async with AsyncSession(engine, expire_on_commit=False) as session: + yield session + + monkeypatch.setattr(auth_module, "create_session", factory) + + +def _snapshot() -> ReservationSnapshot: + return ReservationSnapshot( + release_id=RESERVATION_ID, + key_hash=KEY_HASH, + billing_key_hash=KEY_HASH, + reserved_msats=RESERVED_MSATS, + ) + + +async def _state(engine: AsyncEngine) -> dict[str, Any]: + async with AsyncSession(engine, expire_on_commit=False) as session: + key = await session.get(ApiKey, KEY_HASH) + row = await session.get(ReservationRelease, RESERVATION_ID) + assert key is not None and row is not None + return { + "balance": key.balance, + "reserved_balance": key.reserved_balance, + "total_spent": key.total_spent, + "status": row.status, + } + + +@pytest.mark.asyncio +async def test_recovery_charges_a_reservation_that_never_committed( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + await _seed(engine) + _bind_sessions(monkeypatch, engine) + + result = await _recover_known_charge(_snapshot(), charge_msats=CHARGE_MSATS) + + assert result.settled is True + assert result.charge_msats == CHARGE_MSATS + assert await _state(engine) == { + "balance": BALANCE_MSATS - CHARGE_MSATS, + "reserved_balance": 0, + "total_spent": CHARGE_MSATS, + "status": "charged", + } + + +@pytest.mark.asyncio +async def test_recovery_does_not_charge_twice_when_the_first_commit_landed( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + """The ambiguity that matters: the charge committed, then the caller saw an error. + + The second attempt must read the row as ``charged`` and stop, not charge the + buyer a second time for one request. + """ + await _seed(engine) + _bind_sessions(monkeypatch, engine) + + first = await _recover_known_charge(_snapshot(), charge_msats=CHARGE_MSATS) + second = await _recover_known_charge(_snapshot(), charge_msats=CHARGE_MSATS) + + assert first.settled is True and second.settled is True + assert await _state(engine) == { + "balance": BALANCE_MSATS - CHARGE_MSATS, + "reserved_balance": 0, + "total_spent": CHARGE_MSATS, + "status": "charged", + } + + +@pytest.mark.asyncio +async def test_recovery_reports_unsettled_when_no_session_can_run( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + await _seed(engine) + + def unavailable() -> Any: + raise RuntimeError("database is down") + + monkeypatch.setattr(auth_module, "create_session", unavailable) + monkeypatch.setattr(auth_module, "SETTLEMENT_RECOVERY_BACKOFF_SECONDS", 0.0) + + result = await _recover_known_charge(_snapshot(), charge_msats=CHARGE_MSATS) + + assert result.settled is False + assert result.charge_msats == 0 + assert "database is down" in result.detail + # Nothing was charged, and the reservation is still owned by settlement: + # the caller must write the amount off, not report a free request. + assert await _state(engine) == { + "balance": BALANCE_MSATS, + "reserved_balance": RESERVED_MSATS, + "total_spent": 0, + "status": "active", + } + + +@pytest.mark.asyncio +async def test_recovery_will_not_charge_a_released_reservation( + engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> None: + await _seed(engine, status="released") + _bind_sessions(monkeypatch, engine) + + result = await _recover_known_charge(_snapshot(), charge_msats=CHARGE_MSATS) + + assert result.settled is False + assert "released" in result.detail + assert (await _state(engine))["balance"] == BALANCE_MSATS