diff --git a/routstr/auth.py b/routstr/auth.py index e993647..0ada853 100644 --- a/routstr/auth.py +++ b/routstr/auth.py @@ -19,6 +19,7 @@ from .core.db import ( ReservationRelease, accumulate_routstr_fee, create_session, + get_reservation_status, ) from .core.errors import build_payment_required_error from .core.settings import settings @@ -58,6 +59,29 @@ payments_logger = get_logger("routstr.payments") # Routstr platform fee constants ROUTSTR_FEE_PERCENT: float = 2.1 ROUTSTR_LN_ADDRESS: str = "routstr-fees@rizful.com" + +# A delivered request whose charge failed to commit gets fresh-session +# attempts before the amount is written off. Three covers a transient +# connection error or a serialization conflict without holding the response +# open through a real outage. +SETTLEMENT_RECOVERY_ATTEMPTS: int = 3 +SETTLEMENT_RECOVERY_BACKOFF_SECONDS: float = 0.05 + + +@dataclass(frozen=True) +class SettlementRecovery: + """What a known charge's recovery settled on. + + ``settled`` False means nothing was charged and the reservation still has + to be released; the caller must then record the amount as a write-off, so + an unsettled request cannot be mistaken for a cheap one. + """ + + settled: bool + charge_msats: int + detail: str + + ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900 # Minimum accrued fee (in sats) before the payout loop sends anything. This is a # floor, not a fixed amount: once reached, the whole accumulated balance is paid. @@ -1252,6 +1276,55 @@ async def _charge_reservation_rows( return True +async def _recover_known_charge( + snapshot: ReservationSnapshot, + *, + charge_msats: int, +) -> "SettlementRecovery": + """Charge a delivered request at most once after its commit was ambiguous. + + The claim is conditional on ``status='active'``, so a retry cannot charge + twice: it either finds the row already ``charged`` — the first commit + landed, and the amount it landed with is the amount we were charging — or + it finds it still ``active``, in which case the first commit did not land + and the retry owns the charge. + + Each attempt opens its own session, because a failed commit leaves the + caller's session unusable. ``settled`` is False only when every attempt + failed or the reservation was already released; the caller must then write + the amount off explicitly rather than treat the request as free. + """ + last_error = "no attempt completed" + for attempt in range(SETTLEMENT_RECOVERY_ATTEMPTS): + try: + async with create_session() as session: + if not await _claim_reservation_for_charge(snapshot, session): + status = await get_reservation_status(session, snapshot.release_id) + if status == "charged": + return SettlementRecovery( + True, charge_msats, "a previous attempt committed" + ) + return SettlementRecovery( + False, 0, f"reservation is {status or 'unreadable'}" + ) + if not await _charge_reservation_rows( + session, + billing_key_hash=snapshot.billing_key_hash, + reserved_msats=snapshot.reserved_msats, + charge_msats=charge_msats, + ): + return SettlementRecovery( + False, 0, "reserved balance no longer matches" + ) + await session.commit() + return SettlementRecovery(True, charge_msats, "retried and committed") + except Exception as error: # noqa: BLE001 - ambiguity is what this resolves + last_error = f"{type(error).__name__}: {error}" + if attempt + 1 < SETTLEMENT_RECOVERY_ATTEMPTS: + await asyncio.sleep(SETTLEMENT_RECOVERY_BACKOFF_SECONDS) + return SettlementRecovery(False, 0, last_error) + + async def _adjust_payment_for_tokens( key: ApiKey, response_data: dict, @@ -1341,15 +1414,67 @@ async def _adjust_payment_for_tokens( extra={"error": str(e), "fee_msats": fee_msats}, ) - async def _commit_settlement(cost: CostData, revenue_msats: int) -> None: + async def _commit_settlement(cost: CostData, revenue_msats: int) -> str: + """Commit the charge, resolving an ambiguous commit instead of leaking it. + + Returns ``committed`` when this session committed, ``recovered`` when a + fresh-session retry committed instead — the caller must then refresh + before reading the key, because the failed commit rolled this session + back — and ``unsettled`` when nothing was charged. The caller then + releases the reservation and records the amount as a write-off, because + a served request that could not be charged is a loss, not a cheap + request. + """ try: await session.commit() - except BaseException: + except asyncio.CancelledError: if terminal_outcome is not None: mark_terminal_outcome_unknown( terminal_outcome, "prepaid_commit_ambiguous" ) raise + except Exception as commit_error: + if terminal_outcome is not None: + mark_terminal_outcome_unknown( + terminal_outcome, "prepaid_commit_ambiguous" + ) + # Release whatever the failed commit left open before another + # session tries to write the same rows: an unrolled-back + # transaction holds its row locks, and the retry would block on + # them instead of resolving the ambiguity. + with suppress(Exception): + await session.rollback() + recovery = await _recover_known_charge( + reservation, charge_msats=revenue_msats + ) + if not recovery.settled: + logger.critical( + "Settlement commit stayed ambiguous; the request was served and is not charged", + extra={ + "key_hash": key_log_hash, + "billing_key_hash": billing_log_hash, + "charge_msats": revenue_msats, + "reservation_id": reservation.release_id, + "error": f"{type(commit_error).__name__}: {commit_error}", + "recovery": recovery.detail, + }, + ) + cost.settlement_status = "unsettled" + return "unsettled" + logger.warning( + "Settlement commit was ambiguous and a fresh session committed the charge", + extra={ + "key_hash": key_log_hash, + "billing_key_hash": billing_log_hash, + "charge_msats": recovery.charge_msats, + "reservation_id": reservation.release_id, + "detail": recovery.detail, + }, + ) + cost.settlement_status = "recovered" + status = "recovered" + else: + status = "committed" if terminal_outcome is not None: recorded_usage = response_data.get("usage") context = replace( @@ -1378,6 +1503,41 @@ async def _adjust_payment_for_tokens( revenue_msats=revenue_msats, usage=recorded_usage, ) + return status + + async def _release_unsettled( + cost: CostData, charge_msats: int, reason: str + ) -> None: + """Release a reservation whose charge could not be committed, and say so. + + The release stops the buyer's funds from being stranded, but it is a + write-off rather than a cheap request: the amount that should have been + charged goes to the payments log under its own finalize type, so an + operator can count what an unsettled request cost instead of inferring + it from a response that looks identical to a free one. + """ + cost.settlement_status = reason + cost.charged_msats = 0 + # A failed commit leaves the session in a pending-rollback state; the + # release has to start from a usable session or it cannot run at all. + with suppress(Exception): + await session.rollback() + await release_reservation_only() + payments_logger.error( + "FINALIZE", + extra={ + "event": "finalize", + "key_hash": key_log_hash, + "billing_key_hash": billing_log_hash, + "model": model, + "cost_reserved": deducted_max_cost, + "cost_charged": 0, + "cost_unsettled": charge_msats, + "input_tokens": cost.input_tokens, + "output_tokens": cost.output_tokens, + "finalize_type": reason, + }, + ) # A flat per-unit charge is priced by the caller; its response carries no # usage object for ``calculate_cost`` to read. @@ -1404,6 +1564,11 @@ async def _adjust_payment_for_tokens( }, ) calculated_cost = unpriced_cost(response_data, usage_presence) + # The amount is unknown, so no charge can be invented without + # overcharging every buyer on a pricing hiccup. The release stays, but + # it is named: a request that settled at zero because it could not be + # priced is not a request that cost nothing. + calculated_cost.settlement_status = "released_unpriced" if not await _claim_reservation_for_charge(reservation, session): # A prior charge or release already owns this reservation. Returning @@ -1431,7 +1596,12 @@ async def _adjust_payment_for_tokens( charge_msats=cost.total_msats, ) if charged: - await _commit_settlement(cost, cost.total_msats) + settlement = await _commit_settlement(cost, cost.total_msats) + if settlement == "unsettled": + await _release_unsettled( + cost, cost.total_msats, "released_unsettled" + ) + return cost.dict() await _stop_reservation_heartbeat(reservation.release_id) if not charged: logger.error( @@ -1444,10 +1614,12 @@ async def _adjust_payment_for_tokens( "model": model, }, ) - cost.charged_msats = 0 - await release_reservation_only() + await _release_unsettled(cost, cost.total_msats, "released_unsettled") else: cost.charged_msats = cost.total_msats + # Refresh unconditionally: a recovered settlement rolled this + # session back, so the ORM instance is expired and a bare + # attribute read would hit the database outside the greenlet. await session.refresh(billing_key) logger.info( "Max cost payment finalized", @@ -1476,7 +1648,11 @@ async def _adjust_payment_for_tokens( "balance": billing_key.balance, "reserved_balance": billing_key.reserved_balance, "total_spent": billing_key.total_spent, - "finalize_type": "max_cost", + "finalize_type": ( + "max_cost" + if cost.settlement_status == "settled" + else cost.settlement_status + ), }, ) return cost.dict() @@ -1528,13 +1704,22 @@ async def _adjust_payment_for_tokens( "model": model, }, ) - cost.charged_msats = 0 - await release_reservation_only() + await _release_unsettled( + cost, total_cost_msats, "released_unsettled" + ) return cost.dict() - await _commit_settlement(cost, total_cost_msats) + settlement = await _commit_settlement(cost, total_cost_msats) + if settlement == "unsettled": + await _release_unsettled( + cost, total_cost_msats, "released_unsettled" + ) + return cost.dict() await _stop_reservation_heartbeat(reservation.release_id) cost.charged_msats = total_cost_msats + # Refresh unconditionally: a recovered settlement rolled this + # session back, so the ORM instance is expired and a bare + # attribute read would hit the database outside the greenlet. await session.refresh(billing_key) await _accumulate_fee(total_cost_msats) payments_logger.info( @@ -1591,8 +1776,9 @@ async def _adjust_payment_for_tokens( }, ) await session.rollback() - cost.charged_msats = 0 - await release_reservation_only() + await _release_unsettled( + cost, total_cost_msats, "released_invariant" + ) return cost.dict() sibling_reserved = observed_reserved - deducted_max_cost chargeable_msats = max(0, observed_balance - sibling_reserved) @@ -1615,9 +1801,17 @@ async def _adjust_payment_for_tokens( await session.rollback() raise RuntimeError("Could not atomically finalize cost overrun") - await _commit_settlement(cost, actual_charge_msats) + settlement = await _commit_settlement(cost, actual_charge_msats) + if settlement == "unsettled": + await _release_unsettled( + cost, actual_charge_msats, "released_unsettled" + ) + return cost.dict() await _stop_reservation_heartbeat(reservation.release_id) + # Refresh unconditionally: a recovered settlement rolled this + # session back, so the ORM instance is expired and a bare + # attribute read would hit the database outside the greenlet. await session.refresh(billing_key) cost.charged_msats = actual_charge_msats if actual_charge_msats < total_cost_msats: @@ -1681,7 +1875,12 @@ async def _adjust_payment_for_tokens( charge_msats=total_cost_msats, ) if charged: - await _commit_settlement(cost, total_cost_msats) + settlement = await _commit_settlement(cost, total_cost_msats) + if settlement == "unsettled": + await _release_unsettled( + cost, total_cost_msats, "released_unsettled" + ) + return cost.dict() await _stop_reservation_heartbeat(reservation.release_id) if not charged: @@ -1695,11 +1894,15 @@ async def _adjust_payment_for_tokens( "model": model, }, ) - cost.charged_msats = 0 - await release_reservation_only() + await _release_unsettled( + cost, total_cost_msats, "released_unsettled" + ) else: cost.total_msats = total_cost_msats cost.charged_msats = total_cost_msats + # Refresh unconditionally: a recovered settlement rolled + # this session back, so the instance is expired and a bare + # attribute read would hit the database outside the greenlet. await session.refresh(billing_key) logger.info( diff --git a/routstr/core/db.py b/routstr/core/db.py index 0b171f5..bead171 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -203,6 +203,27 @@ async def _transition_stale_reservation( return bool(transition.rowcount == 1) +async def get_reservation_status(session: AsyncSession, release_id: str) -> str | None: + """Read one durable reservation's state, or None when it cannot be read. + + Settlement recovery has to tell three cases apart after an ambiguous + commit: our charge landed (``charged``), the row is still ours to charge + (``active``), or something else already terminalized it (``released``). + Returning None rather than raising keeps the caller's decision explicit: + an unreadable row is not evidence that nothing was charged. + """ + try: + return ( + await session.exec( + select(ReservationRelease.status).where( + col(ReservationRelease.id) == release_id + ) + ) + ).first() + except Exception: + return None + + async def _release_legacy_aggregate( session: AsyncSession, key_hash: str, diff --git a/routstr/payment/cost_calculation.py b/routstr/payment/cost_calculation.py index f3eac48..4a7b010 100644 --- a/routstr/payment/cost_calculation.py +++ b/routstr/payment/cost_calculation.py @@ -53,6 +53,10 @@ class CostData(BaseModel): cache_creation_source: str = Field(default="missing", exclude=True) # Actual debit after finalization; None means settlement has not run yet. charged_msats: int | None = None + # How this request's charge was settled, for the ledger and the operator. + # ``settled`` is the ordinary case; the other values name a failure that + # used to be indistinguishable from a request that cost nothing. + settlement_status: str = Field(default="settled", exclude=True) upstream_usd: float = Field(default=0.0, exclude=True) diff --git a/routstr/upstream/base.py b/routstr/upstream/base.py index b138aa8..f209fcb 100644 --- a/routstr/upstream/base.py +++ b/routstr/upstream/base.py @@ -123,6 +123,7 @@ if typing.TYPE_CHECKING: from .ehbp import ConfidentialInferenceProfile, EHBPForwardingTarget logger = get_logger(__name__) +payments_logger = get_logger("routstr.payments") # A stored catalog younger than this share of the refresh interval is reused # instead of fetched again, so N workers cost roughly one upstream fetch per @@ -1461,6 +1462,24 @@ class BaseUpstreamProvider: "reserved_balance": snapshot.reserved_msats, }, ) + else: + # A release here happens after the upstream answered: the + # request was served and nothing will be charged for it. That + # is a write-off, not a cheap request, so it is recorded where + # the accounting reads. The amount is the reservation — the + # ceiling the request could have been charged — because the + # settlement that would have known the real cost failed. + payments_logger.error( + "FINALIZE", + extra={ + "event": "finalize", + "key_hash": key.hashed_key[:8] + "...", + "cost_reserved": snapshot.reserved_msats, + "cost_charged": 0, + "cost_unsettled": snapshot.reserved_msats, + "finalize_type": "released_unsettled", + }, + ) # A failed release remains recoverable by the stale-reservation # sweep. Retrying settlement here could charge after an ambiguous # database failure or replace the original stream exception.