Files
Crussell/backend/handlers/payments/sweep.go
T
popertots faceb9809c fix: review-loop A — discount credit on admin payments, campaign over-credit cap, sweep replay window, dedup refund revalidation, duplication/modularisation, GBP pence naming
Round-A fresh review (6 agents) + fix + secondary cross-cutting + verification rounds:
- F1: campaign discounts reduce the charged amount (deposit credit + admin PaymentModal discounted total); capDiscountToRemainingObligation prevents over-credit at completion in all four campaign blocks
- F2: sweep replay rescue distinguishes legitimate same-key retries (21h window) from expired-key new charges; ccof blind-fails leave pending + CRITICAL instead of clawing back
- F3: post-start online overflow carved as a tip record (mirrors terminal split builder)
- A1: single-source Square decline-code classification (till delegates to square.IsDefinitivePaymentError)
- A2/A5: refund attempt-cap literals consolidated; refund-failure counter capped + reset on terminal resolutions + admin notifications
- A3/A9: idempotency helpers adopted across derivations; IsExplicitDevOrMockEnv relocated + all gates unified (incl. health-check)
- A7: 2FA user+IP limiter + TRUST_PROXY_HEADERS startup warning; SNAPSHOT_ENC_KEY startup validation; TWO_FACTOR_PEPPER docs corrected
- A8: snapshot encryption on all 6 write sites + marker-aware reuse paths; MPV->SPV effective voucher type (single VAT point)
- A10/A11/A12/A16: gift-card slot scan advances past failed; amount-aware refund reconciliation; completed-booking refund re-check; PaymentWasRefunded on SquareClient interface
- Dedup refund revalidation on tip/terminal/gift-card paths; sweep acknowledged_at IS NULL parity; refund-notification single source (exported payments.InsertRefundFailedNotifications)
- Duplication/modularisation round: shared frontend helpers (sanitizeDecimalInput, campaignDiscountCents, twoFactorBlocksSavedCards getter, generateUUID), single-source MaxIdempotencyKeyLength, notification-helper consolidation, snapshot-guard comments
- Cross-cutting GBP rename: Cents->Pence across backend + frontend + tests (26 identifiers, 16 files)
- Tests: 11 behavior-change tests updated to new invariants; coverage for fixed functions; frontend vitest 55 tests; docs corrected (test counts, 2FA delivery, pre-launch checklist, resolution status)
- gitleaks: allowlist backend/internal/square test fixtures (mock idempotency keys)

All 25 backend packages pass; frontend 55/55 + build clean; env-docs 41/41.
2026-08-22 00:34:50 +01:00

1860 lines
94 KiB
Go

package payments
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log"
"log/slog"
"math"
"strconv"
"strings"
"time"
"crussell/clock"
"crussell/db"
"crussell/internal/square"
"github.com/jackc/pgx/v5"
)
// SweepStalePendingPayments resolves pending payment records that are older
// than Square's idempotency-key retention window (~24h). A pending record
// means the DB committed but the Square charge outcome is unknown; it
// normally resolves on a same-key client retry. But if the client abandoned
// the attempt, the record stays pending forever — and retrying it after the
// key expires would ISSUE A SECOND CHARGE (Square no longer dedups). Failing
// stale pendings closes that double-charge window: a late retry finds a
// 'failed' record and stops instead of charging again.
//
// Before failing a row, the sweep reconciles it against Square: a
// genuinely-charged row (Square success, DB post-charge failure) with a
// square_payment_id is rescued to 'completed' instead of being swept to
// 'failed' with no automatic resolution — the money would otherwise be lost
// in limbo (MINOR-R3). Reconciliation is deliberately minimal: status +
// updated_at only, no split/VAT recomputation (that is the handler's job; the
// row is >24h stale and this is a reconciliation rescue). A row with NO
// square_payment_id but a STORED idempotency key (the lost-response case — the
// charge may have completed at Square with the response never received) is
// reconciled at Square by replaying the key (ReplayPaymentByKey) at an EARLIER
// 22h cutoff, while the key is still inside Square's ~24h retention window: a
// COMPLETED charge is rescued to 'completed' with the real square_payment_id
// written back, and a charge Square proves never happened is failed. Rows with
// neither a square_payment_id nor a stored key cannot be reconciled and are
// failed with a WARN exactly as the legacy sweep did.
//
// Only online/till card payments can be pending — cash/giftcard/on_the_house
// are committed synchronously and never enter this state. Both the payments
// table and till_sales carry pending card-sale rows and are swept here.
const stalePendingPaymentAge = 24 * time.Hour
// stalePendingKeyedAge is how old a pending row with a stored idempotency key
// (but no square_payment_id — the lost-response case) must be before the sweep
// reconciles it at Square by replaying the key. It is deliberately 2h EARLIER
// than stalePendingPaymentAge so the replay lands comfortably inside Square's
// ~24h idempotency-key retention window: replaying at exactly 24h risks the key
// already having expired, and an expired key would make the probe rejection
// look like "never charged" even when the charge actually landed. Rows older
// than stalePendingPaymentAge when swept can no longer be replayed
// trustworthily and fall back to the legacy blind-fail + WARN.
const stalePendingKeyedAge = 22 * time.Hour
// SweepStalePendingPayments resolves stale pending payments and till sales; see the rationale block on stalePendingPaymentAge above.
func SweepStalePendingPayments(ctx context.Context) (int, error) {
cutoff := clock.Now().Add(-stalePendingPaymentAge)
keyedCutoff := clock.Now().Add(-stalePendingKeyedAge)
// Pass 0: stranded Square-less MANUAL refund rows at the attempt cap
// (maxManualRefundAttempts). The Square-less pre-pass inside
// sweepManualPendingSquareRefunds (refunds.go) reconciles only rows with
// refund_attempts < maxManualRefundAttempts; legacy rows
// that already hit the cap are never reconciled by it and would stay
// 'pending' forever, permanently blocking the over-refund guard. Such rows
// can never be refunded via Square (no square_payment_id), so they are
// marked 'failed' + admin-notified here for in-person arrangement (F10).
sqlessRefundCount, sqlessErr := sweepSquarelessManualRefundsAtAttemptCap(ctx)
if sqlessErr != nil {
log.Printf("Failed to reconcile Square-less manual refunds at the 3-attempt cap: %v", sqlessErr)
}
// Pass 1 (earlier cutoff): rows with a stored idempotency key but NO
// square_payment_id are the lost-response case — the charge may have landed
// at Square with the response lost. They are reconciled at Square by
// replaying the key (ReplayPaymentByKey) while the key is still inside
// Square's ~24h retention window: a COMPLETED charge is rescued to
// 'completed' with the real square_payment_id, a charge Square proves never
// happened is failed (and a till sale's funded gift card clawed back), an
// ambiguous answer leaves the row pending for the next run, and a row
// already past the retention window is blind-failed with a WARN exactly as
// the legacy sweep did (replaying an expired key would misread the probe
// rejection as "never charged").
payKeyedCount, payKeyedCompleted, payKeyedUnverified, err := sweepKeyedStaleRows(ctx, "payments", keyedCutoff)
if err != nil {
return 0, err
}
tillKeyedCount, tillKeyedCompleted, tillKeyedUnverified, err := sweepKeyedStaleRows(ctx, "till_sales", keyedCutoff)
if err != nil {
return 0, err
}
// Pass 2 (legacy 24h cutoff): rows WITH a square_payment_id are reconciled
// by payment id; rows with neither a payment id nor a stored key cannot be
// reconciled and are failed directly (WARN — the charge outcome is unknown).
payCount, payCompleted, err := sweepStaleRows(ctx, "payments", cutoff)
if err != nil {
return 0, err
}
tillCount, tillCompleted, err := sweepStaleRows(ctx, "till_sales", cutoff)
if err != nil {
return 0, err
}
total := sqlessRefundCount + payKeyedCount + tillKeyedCount + payCount + tillCount
payTotal := payKeyedCount + payCount
tillTotal := tillKeyedCount + tillCount
completed := payKeyedCompleted + tillKeyedCompleted + payCompleted + tillCompleted
if total > 0 {
log.Printf("[SWEEP] Resolved %d stale pending payments (%d payments, %d till sales) older than %s — late retries will be rejected, preventing a second Square charge; %d reconciled to completed against Square (%d payments, %d till sales)", total, payTotal, tillTotal, stalePendingPaymentAge, completed, payKeyedCompleted+payCompleted, tillKeyedCompleted+tillCompleted)
}
// Routine sweep bookkeeping, not an incident: failing stale pending rows is
// the sweep's DESIGNED behaviour. A row swept to failed may still have been
// charged at Square with a lost response, so note it at WARN level for an
// admin doing a periodic money reconciliation — but this fires on every
// normal run and must not be elevated to CRITICAL (which is reserved for
// genuinely unrecoverable post-charge branches). Rows the keyed reconcile
// PROVED never charged are deliberately excluded from the WARN (they cannot
// have moved money); only rows failed without a reconcile proof — keyless
// rows and keyed rows past the retention window — are counted.
if payKeyedUnverified > 0 {
log.Printf("[SWEEP] WARN: %d pending payments with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response — verify before refunding/charging", payKeyedUnverified)
}
if tillKeyedUnverified > 0 {
log.Printf("[SWEEP] WARN: %d pending till sales with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response", tillKeyedUnverified)
}
if payCount > 0 {
log.Printf("[SWEEP] WARN: %d pending payments marked failed may have been charged at Square with a lost response — verify before refunding/charging", payCount-payCompleted)
}
if tillCount > 0 {
log.Printf("[SWEEP] WARN: %d pending till sales marked failed may have been charged at Square with a lost response", tillCount-tillCompleted)
}
return total, nil
}
// staleRow is one stale pending row read by the sweep so it can reconcile
// rows that carry a Square reference BEFORE failing them. For till_sales rows
// the gift-card context needed to claw back funded money on a definitive
// failure is carried alongside: the card id, who it was redeemed to, whether
// this sale created the card (created_at equality — provable because both
// timestamps are the transaction-start NOW() in the same tx), and the sale
// amount (the exact funding this sale added).
type staleRow struct {
ID string
SquarePaymentID string
// IdempotencyKey is the deterministic Square idempotency key stored on the
// pending row ("" when absent). Rows WITH a key but NO square_payment_id are
// the lost-response case: the sweep replays the key at Square to learn the
// true charge outcome before declaring failure.
IdempotencyKey string
// SquareSourceID is the row's CURRENT square_source_id — the source_id sent
// in the latest CreatePayment attempt, refreshed whenever a same-key retry
// reuses the pending row. The replay-by-key overrides the stored snapshot's
// embedded source with this value so the replayed body matches the source
// the retained key actually used. Replaying a different body would return
// IDEMPOTENCY_KEY_REUSED, which proves nothing about whether the charge
// landed.
SquareSourceID string
// SquareRequestSnapshot is the verbatim original CreatePayment request JSON
// stored on the row at charge time (square_request_snapshot) — the FULL
// body the replay-by-key must repeat (source, key, amount, customer_id,
// reference_id, note, buyer_email_address, verification_token, ...).
// Square's idempotency dedup compares the whole request, so a replay built
// from partial row data returns IDEMPOTENCY_KEY_REUSED for a RETAINED key
// and the row stays pending forever (safe but never auto-rescued). Nil for
// legacy rows — the sweep then rebuilds the minimal body from the stored
// key + source + amount.
SquareRequestSnapshot []byte
// AmountPence is the row's charge amount in pence — the amount the original
// CreatePayment used. The replay-by-key must repeat it so Square's
// idempotency dedup returns the original payment.
AmountPence int64
// CreatedAt is the pending row's creation time; rows already past Square's
// idempotency-key retention window cannot be replayed trustworthily.
CreatedAt time.Time
// BookingID is the payments row's booking_id ("" when the row has none).
// A payments row with NO booking is a gift-card purchase (BuyGiftCard
// inserts without a booking): rescuing it to 'completed' on a Square
// COMPLETED reconcile would permanently block the same-key retry that
// delivers the card, so such rows are kept pending instead (C6).
BookingID *string
// CreatedBy is the payments row's created_by user id (gift-card purchases
// always carry the purchaser), used to attribute the critical-payment admin
// notification.
CreatedBy *string
ItemID string // till_sales.item_id — the gift card ("" when NULL / non-gift-card)
RedeemToUserID *string // gift_cards.redeemed_by — user credited by a create-with-redeem
IsCreate bool // true when this sale created the gift card (timestamps equal)
HasGiftCard bool // false when the LEFT JOIN found no gift_cards row (gc.id IS NULL)
TotalAmount float64 // till_sales.total_amount — the funding this sale added
}
// sweepStaleRows resolves the stale pending rows of one table. Rows with a
// square_payment_id are reconciled at Square first (COMPLETED → 'completed',
// anything else → 'failed' exactly as the legacy bulk UPDATE did); rows
// without one cannot be reconciled and are failed directly. A till_sales row
// whose reconcile PROVES the charge never completed (NOT_FOUND / non-COMPLETED)
// also claws back the funded gift card atomically with the failed mark; the
// blind-fail path (no square_payment_id — the charge may have landed) never
// claws back. Returns the total rows resolved and how many were rescued to
// 'completed'.
func sweepStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved int, completed int, err error) {
switch table {
case "payments", "till_sales":
default:
return 0, 0, fmt.Errorf("sweep: unknown stale table %q", table)
}
stale, err := fetchStaleRows(ctx, table, cutoff, false)
if err != nil {
return 0, 0, err
}
for _, r := range stale {
if r.SquarePaymentID != "" {
switch reconcileStalePaymentAtSquare(ctx, table, r.SquarePaymentID) {
case staleReconcileCompleted:
if table == "payments" && r.BookingID == nil {
// Gift-card purchase row: the charge landed at Square but
// the gift card was never delivered (C6). Completing the row
// permanently blocks the same-key retry that delivers the
// card — leave it pending and alert.
leaveGiftCardPurchasePending(ctx, r)
continue
}
// F3: a charge Square reports COMPLETED must never be
// completed on a booking that was cancelled during the pending
// window — the cancellation refund path computes refunds from
// completed payments and would miss it, charging the customer
// with NO automatic refund. Re-read bookings.status (FOR
// UPDATE) and refuse on a cancelled booking. Rows with no
// booking (gift-card purchases, till_sales) are never gated.
if table == "payments" {
switch gateStalePaymentRescueOnBooking(ctx, r) {
case staleBookingGateRefused:
resolved++
continue
case staleBookingGateUnknown:
continue
}
}
if rescueStaleRowCompleted(ctx, table, r.ID) {
resolved++
completed++
}
continue
case staleReconcileLeavePending:
// Square's answer was ambiguous (transport/5xx) — the charge
// may still be in flight at Square. Do NOT touch the row: the
// next sweep run reconciles it again, and a same-key retry
// must still be able to reuse the pending row if the charge
// actually completed.
log.Printf("Stale pending %s row %s left pending (Square reconcile ambiguous) — will retry next sweep", table, r.ID)
continue
}
// staleReconcileDefinitivelyFailed falls through to the fail path.
}
// Fail path. A till-sale reconcile that PROVED the charge never
// completed (NOT_FOUND / non-COMPLETED) claws back the funded gift
// card atomically with the failed mark (HIGH-1). The blind-fail path
// (no square_payment_id — the charge outcome is unknown) NEVER claws
// back: the money may have landed at Square.
if table == "till_sales" && r.SquarePaymentID != "" && r.HasGiftCard {
if clawbackTillSaleFunding(ctx, r) {
resolved++
}
continue
}
if failStaleRow(ctx, table, r.ID) {
resolved++
// A5a: blind-failing a row with NO square_payment_id leaves the
// charge outcome unknown (the money may have landed at Square with a
// lost response), so surface it in the admin notification centre.
// Rows that reach this line WITH a square_payment_id were PROVED
// never-charged by the by-id reconcile and need no alert.
if r.SquarePaymentID == "" {
notifyStaleRowCritical(ctx, r)
}
}
}
return resolved, completed, nil
}
// sweepKeyedStaleRows resolves stale pending rows that carry a stored
// idempotency key but no square_payment_id — the lost-response case, where the
// charge may have completed at Square with the response never reaching the
// app. Such rows are reconciled at Square by replaying the key while it is
// still inside Square's ~24h retention window (the earlier stalePendingKeyedAge
// cutoff guarantees this):
//
// - a COMPLETED payment under the key → rescued to 'completed' with the real
// square_payment_id written back (the customer WAS charged);
// - a payment Square proves never happened (unknown key / FAILED status) →
// marked 'failed', and a till sale's funded gift card is clawed back
// (reconcile PROVED the funding has no charge behind it);
// - an ambiguous answer (transport/5xx) → left pending for the next run;
// - a row already older than the retention window when swept → cannot be
// replayed trustworthily (an expired key would misread the probe rejection
// as "never charged" even when the charge landed), so it is blind-failed
// with a WARN exactly as the legacy sweep did.
//
// Returns the total rows resolved, how many were rescued to 'completed', and
// how many were marked failed WITHOUT a reconcile proof (blind-fail WARNs).
func sweepKeyedStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved, completed, unverifiable int, err error) {
switch table {
case "payments", "till_sales":
default:
return 0, 0, 0, fmt.Errorf("sweep: unknown stale table %q", table)
}
stale, err := fetchStaleRows(ctx, table, cutoff, true)
if err != nil {
return 0, 0, 0, err
}
// Square's idempotency-key retention window closes at stalePendingPaymentAge;
// a row older than that when swept can no longer be replayed trustworthily.
replayExpired := clock.Now().Add(-stalePendingPaymentAge)
for _, r := range stale {
if r.CreatedAt.Before(replayExpired) {
// Key retention window already closed — replaying would misread an
// expired key as "never charged". Blind-fail + WARN exactly as the
// legacy sweep did; a till sale's funded gift card is NOT clawed
// back (the charge outcome is unknown, the money may have landed).
// A5a: this blind-fail can be hiding a real charge (the lost
// response the stored key was meant to reconcile), so the admin
// notification centre must surface it too.
if failStaleRow(ctx, table, r.ID) {
resolved++
unverifiable++
notifyStaleRowCritical(ctx, r)
}
log.Printf("Stale pending %s row %s has a stored idempotency key but is already past Square's key retention window — marked failed without a replay reconcile (may have been charged with a lost response)", table, r.ID)
continue
}
switch res, sqPayID := reconcileStalePaymentByKey(ctx, table, r); res {
case staleReconcileCompleted:
if table == "payments" && r.BookingID == nil {
// Gift-card purchase row (C6): the charge landed at Square but
// the card was never delivered. Completing the row blocks the
// same-key retry that delivers the card — leave it pending.
leaveGiftCardPurchasePending(ctx, r)
continue
}
// F3: same money-safety gate as the by-id rescue above — a charge
// Square reports COMPLETED must never be completed on a booking
// that was cancelled during the pending window.
if table == "payments" {
switch gateStalePaymentRescueOnBooking(ctx, r) {
case staleBookingGateRefused:
resolved++
continue
case staleBookingGateUnknown:
continue
}
}
if rescueKeyedStaleRowCompleted(ctx, table, r.ID, sqPayID) {
resolved++
completed++
}
case staleReconcileLeavePending:
// Ambiguous replay — the charge may still be in flight. Leave the
// row pending; the next sweep run reconciles it again.
log.Printf("Stale pending %s row %s left pending (replay-by-key reconcile ambiguous) — will retry next sweep", table, r.ID)
case staleReconcileDefinitivelyFailed:
// Square proved the charge never happened (key unknown / FAILED) —
// a till sale's funded gift card is clawed back atomically with the
// failed mark, unlike the blind-fail path where the outcome is unknown.
if table == "till_sales" && r.HasGiftCard {
if clawbackTillSaleFunding(ctx, r) {
resolved++
}
continue
}
if failStaleRow(ctx, table, r.ID) {
resolved++
}
}
}
return resolved, completed, unverifiable, nil
}
// fetchStaleRows reads the stale pending rows of one table that are older than
// cutoff. keyedOnly restricts the query to rows that can be reconciled by
// replaying a stored idempotency key: those with a key but no square_payment_id
// (rows WITH a square_payment_id are reconciled by payment id in the main
// pass). The legacy till_sales sweep only touches card methods — cash and
// on_the_house are committed synchronously and never pending, but the
// predicate is kept so behaviour is byte-identical for any unexpected row.
func fetchStaleRows(ctx context.Context, table string, cutoff time.Time, keyedOnly bool) ([]staleRow, error) {
keyedPredicate := ""
if keyedOnly {
keyedPredicate = ` AND idempotency_key IS NOT NULL AND square_payment_id IS NULL`
}
methodFilter := ""
if table == "till_sales" {
methodFilter = ` AND payment_method IN ('online_square', 'in_person_card')`
}
var rows pgx.Rows
var err error
if table == "till_sales" {
rows, err = db.Conn.Query(ctx, `
SELECT ts.id, COALESCE(ts.square_payment_id, ''), COALESCE(ts.idempotency_key, ''),
COALESCE(ts.square_source_id, ''), COALESCE(ts.square_request_snapshot, ''),
ts.created_at, ts.item_id, gc.redeemed_by,
(ts.created_at = gc.created_at) AS is_create,
(gc.id IS NOT NULL) AS has_gift_card, ts.total_amount
FROM till_sales ts
LEFT JOIN gift_cards gc ON gc.id = ts.item_id
WHERE ts.status = 'pending' AND ts.created_at < $1`+methodFilter+keyedPredicate+`
`, cutoff)
} else {
// table is an internal constant ("payments"), never user input, but the
// identifier is routed through pgx.Identifier.Sanitize — the same
// treatment failStaleRow / rescueStaleRowCompleted give the table name —
// so no raw, unquoted table name is ever concatenated into the statement.
rows, err = db.Conn.Query(ctx, `
SELECT id, COALESCE(square_payment_id, ''), COALESCE(idempotency_key, ''),
COALESCE(square_source_id, ''), COALESCE(square_request_snapshot, ''),
created_at, amount, booking_id, created_by
FROM `+pgx.Identifier{table}.Sanitize()+`
WHERE status = 'pending' AND created_at < $1`+keyedPredicate+`
`, cutoff)
}
if err != nil {
return nil, err
}
defer rows.Close()
var stale []staleRow
for rows.Next() {
r, scanErr := scanStaleRow(table, rows)
if scanErr != nil {
log.Printf("Failed to scan stale pending row from %s: %v", table, scanErr)
continue
}
stale = append(stale, r)
}
return stale, rows.Err()
}
// scanStaleRow scans one row produced by fetchStaleRows into a staleRow,
// deriving the replay amount (in pence) from the stored pound figure — the
// original CreatePayment amount the replay-by-key must repeat.
func scanStaleRow(table string, rows pgx.Rows) (staleRow, error) {
var r staleRow
if table == "till_sales" {
var itemID, redeemedBy, snapshot sql.NullString
var isCreate *bool
var hasGiftCard bool
if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt,
&itemID, &redeemedBy, &isCreate, &hasGiftCard, &r.TotalAmount); err != nil {
return r, err
}
r.SquareRequestSnapshot = []byte(snapshot.String)
r.ItemID = itemID.String
if redeemedBy.Valid && redeemedBy.String != "" {
r.RedeemToUserID = &redeemedBy.String
}
r.IsCreate = isCreate != nil && *isCreate
r.HasGiftCard = hasGiftCard
r.AmountPence = int64(math.Round(r.TotalAmount * 100))
return r, nil
}
var amount float64
var bookingID, createdBy, snapshot sql.NullString
if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &amount, &bookingID, &createdBy); err != nil {
return r, err
}
r.SquareRequestSnapshot = []byte(snapshot.String)
r.AmountPence = int64(math.Round(amount * 100))
if bookingID.Valid && bookingID.String != "" {
b := bookingID.String
r.BookingID = &b
}
if createdBy.Valid && createdBy.String != "" {
c := createdBy.String
r.CreatedBy = &c
}
return r, nil
}
// failStaleRow marks one stale pending row 'failed'. Returns true when the row
// was updated (status was still 'pending').
func failStaleRow(ctx context.Context, table, id string) bool {
// table is an internal constant ("payments"/"till_sales"), never user
// input, but the identifier is routed through pgx.Identifier.Sanitize so no
// raw, unquoted table name is ever concatenated into the statement.
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, id)
if err != nil {
log.Printf("Failed to mark stale pending row %s failed: %v", id, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// rescueStaleRowCompleted marks one stale pending row (whose square_payment_id
// already resolves to a COMPLETED charge) 'completed'. Returns true when the
// row was updated.
func rescueStaleRowCompleted(ctx context.Context, table, id string) bool {
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, id)
if err != nil {
log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// rescueKeyedStaleRowCompleted marks a keyed stale pending row 'completed' with
// the square_payment_id returned by the replay — the lost-response rescue. The
// reconcile is deliberately minimal (status + square_payment_id + updated_at
// only, no split/VAT recomputation): the row is >22h stale and this is a
// reconciliation rescue, mirroring the F3 by-id rescue's minimality. Returns
// true when the row was updated.
func rescueKeyedStaleRowCompleted(ctx context.Context, table, id, squarePaymentID string) bool {
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', square_payment_id = $1, updated_at = NOW()
WHERE id = $2 AND status = 'pending'
`, squarePaymentID, id)
if err != nil {
log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// staleBookingGate is the outcome of the pre-completion booking-status recheck
// for a stale pending payment row whose charge Square reports COMPLETED.
type staleBookingGate int
const (
// staleBookingGateAllowed — the booking is still payable; the rescue may proceed.
staleBookingGateAllowed staleBookingGate = iota
// staleBookingGateRefused — the booking is cancelled/lapsed/no-show; the row
// was marked FAILED and a critical admin notification raised.
staleBookingGateRefused
// staleBookingGateUnknown — the booking status could not be read; the row was
// left pending (never completed on an unknown state).
staleBookingGateUnknown
)
// gateStalePaymentRescueOnBooking re-reads the booking of a stale pending
// payment row before the sweep rescues it to 'completed' on a Square COMPLETED
// reconcile. It mirrors the live-path guard bookingStatusAllowsCompletedPayment
// (handlers.go): a charge that lands AFTER the booking was cancelled must never
// be silently completed — the cancellation refund path computes refunds from
// completed payments and would miss it, charging a customer for a cancelled
// booking with NO automatic refund (F3).
//
// The booking status is re-read FOR UPDATE inside a transaction so the
// decision serializes against a concurrent cancellation (C5 pattern, mirrors
// recordTerminalPaymentTx). On a cancelled/lapsed/no-show booking the payment
// row is marked FAILED (not completed) and a critical admin notification is
// raised, atomically with the decision, so an operator refunds the customer
// manually. A booking whose status cannot be read is never completed — the row
// is left pending for the next sweep run. Rows with no booking (till_sales;
// gift-card purchases are diverted by the callers before this helper) are
// never gated.
func gateStalePaymentRescueOnBooking(ctx context.Context, r staleRow) staleBookingGate {
if r.BookingID == nil {
return staleBookingGateAllowed
}
tx, err := db.Conn.Begin(ctx)
if err != nil {
log.Printf("CRITICAL: failed to begin booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err)
return staleBookingGateUnknown
}
defer func() {
if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
log.Printf("Failed to rollback booking-status recheck for stale pending payment %s: %v", r.ID, err)
}
}()
var status string
if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, *r.BookingID).Scan(&status); err != nil {
// Booking gone or unreadable: the money state is unknown. Never
// complete on an unknown state — a completed payment on a vanished
// booking would strand the charge outside the refund system.
log.Printf("CRITICAL: Square reports stale pending payment %s COMPLETED but re-reading booking %s status failed (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err)
return staleBookingGateUnknown
}
if bookingStatusAllowsCompletedPayment(status) {
// Payable — release the booking lock; the caller rescues the row.
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("CRITICAL: failed to commit booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, cErr)
return staleBookingGateUnknown
}
return staleBookingGateAllowed
}
// Cancelled / lapsed / no-show booking: completing the payment would
// charge a customer for a booking the cancellation flow already closed,
// with NO automatic refund. Mark the row FAILED (not completed) and alert
// ops so an operator refunds the customer manually.
tag, upErr := tx.Exec(ctx, `
UPDATE `+pgx.Identifier{"payments"}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, r.ID)
if upErr != nil {
log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — marking the row failed errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, upErr)
return staleBookingGateUnknown
}
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — committing the failed mark errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, cErr)
return staleBookingGateUnknown
}
if int(tag.RowsAffected()) > 0 {
insertCriticalPaymentNotification(ctx, r.BookingID, r.CreatedBy)
log.Printf("CRITICAL: stale pending payment %s is COMPLETED at Square but booking %s is %q — payment marked FAILED instead of completed; operator must refund the customer manually", r.ID, *r.BookingID, status)
return staleBookingGateRefused
}
// The row was already resolved concurrently — nothing left to refuse.
return staleBookingGateUnknown
}
// clawbackTillSaleFunding claws back a stale pending till sale's funded gift
// card (atomically with the failed mark) after a reconcile PROVED the charge
// never completed — the funding has no charge behind it. A sale with no gift
// card is only marked failed. Returns true when the sale was resolved to
// failed; false when it was already resolved by someone else, had no gift card,
// or the clawback failed (CRITICAL logged).
func clawbackTillSaleFunding(ctx context.Context, r staleRow) bool {
action := "topup"
if r.IsCreate {
action = "create"
}
if revErr := revertGiftCardFunding(ctx, action, r.ItemID, r.TotalAmount, r.RedeemToUserID, r.ID); revErr != nil {
if errors.Is(revErr, errTillSaleNotPending) {
log.Printf("Stale pending till sale %s was already resolved (not pending) — skipping funding clawback", r.ID)
return false
}
log.Printf("CRITICAL: failed to claw back gift card %s funding for stale till sale %s: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", r.ItemID, r.ID, revErr)
return false
}
return true
}
// replayLegitimateRetryWindow is the maximum lag between the pending row's
// creation and a replayed payment's creation for the payment to be the REAL
// charge under a legitimately replayed key. A same-key retry — the documented
// retry path (handlers.go:1579-1591) — creates its charge somewhere between
// the row's creation and the sweep's 22h keyed cutoff (stalePendingKeyedAge),
// so any COMPLETED payment created within [row.CreatedAt, row.CreatedAt +
// replayLegitimateRetryWindow] can be that retry charge and must be rescued.
// A payment created LATER than 21h after the row (i.e. within ~1-3h of the
// sweep's own replay, which runs at row age 22h+) is the classic expired-key
// replay-induced charge — the sweep just created it by replaying the still
// valid saved-card source under a key Square no longer retains — and rescuing
// it would hide the duplicate charge behind the original row (finding A1).
// 21h is a clear margin: a legitimate retry cannot occur after the sweep has
// already picked the row up at the 22h cutoff.
const replayLegitimateRetryWindow = 21 * time.Hour
// replayMatchesRowAmount reports whether the replayed payment charged the same
// amount the pending row records — the amount the sweep's replay body repeats
// and the amount any same-key retry MUST reuse (the retry path rejects a
// different amount). A replayed payment carrying a DIFFERENT amount cannot be
// the charge this row is waiting on and must never be rescued onto it. A
// payment with no amount (zero — test fixtures; real Square payments always
// carry one) is not refused here: the lag window below is the primary guard.
func replayMatchesRowAmount(r staleRow, pr *square.PaymentResult) bool {
return pr == nil || pr.Amount == 0 || pr.Amount == r.AmountPence
}
// replayWithinLegitimateWindow reports whether a replayed COMPLETED payment is
// the REAL charge this pending row is waiting on — the ORIGINAL charge under a
// retained key (created ~at row creation) or a later SAME-KEY RETRY charge
// (created between the row's creation and the 22h sweep cutoff, F2). The
// amount must match the row (a retry can never change it) and the payment must
// have been created within replayLegitimateRetryWindow of the row. The source
// is matched by construction: the replay body is rebuilt from the row's stored
// square_request_snapshot with the LIVE square_source_id override, so a payment
// returned by the replay necessarily charged the row's source (Square's
// PaymentResult does not echo the source id back, so it cannot be compared
// directly). A payment created very near the sweep time (lag > 21h) is the
// expired-key replay-induced charge and is NOT legitimate.
func replayWithinLegitimateWindow(r staleRow, pr *square.PaymentResult) bool {
if !replayMatchesRowAmount(r, pr) {
return false
}
created, ok := parseReplayedCreatedAt(pr)
if !ok || r.CreatedAt.IsZero() {
return false
}
return !created.Before(r.CreatedAt) && !created.After(r.CreatedAt.Add(replayLegitimateRetryWindow))
}
// isSavedCardSource reports whether a Square source id is a card-on-file
// (saved-card) reference. Only a ccof: source stays valid for recharging long
// after the charge attempt that stored it: a cnon: nonce is single-use, so an
// expired-key replay of a cnon: source is always rejected (proof the charge
// never happened), while a replay of a still-valid ccof: source can land a NEW
// charge under the expired key.
func isSavedCardSource(source string) bool {
return strings.HasPrefix(source, "ccof:")
}
// parseReplayedCreatedAt parses a PaymentResult's ISO 8601 CreatedAt into the
// instant the replayed payment was created at Square. Real Square always
// returns created_at on a payment, so an empty or unparseable value is a
// client/response anomaly.
func parseReplayedCreatedAt(pr *square.PaymentResult) (time.Time, bool) {
if pr == nil || pr.CreatedAt == "" {
return time.Time{}, false
}
created, err := time.Parse(time.RFC3339, pr.CreatedAt)
if err != nil {
return time.Time{}, false
}
return created, true
}
// replayRevealsNewCharge reports whether a COMPLETED keyed replay returned a
// NEW charge rather than the ORIGINAL payment under a retained idempotency key
// (finding A1). A retained-key dedup returns the original payment, created at
// the same instant the pending row was created; a NEW charge made by an
// expired-key replay (Square's ~24h key retention is UNVERIFIED —
// square_http_client.go:626) against the still-valid ccof: source is created
// ~22h later. A same-key RETRY (handlers.go:1579-1591) is a legitimate
// exception: the retry's charge is created between the row's creation and the
// 22h sweep cutoff, so a replayed payment inside replayLegitimateRetryWindow
// is the REAL charge and must be rescued (F2). Refusal is money-safe: a
// replayed payment that cannot be proven to be the original (or a retry
// within the legitimate window) is never rescued (the row stays pending, a
// CRITICAL log is raised and an admin notification inserted), so a hidden
// second charge can never masquerade as the original one.
//
// The check runs ONLY against real Square timestamps: it is gated off in an
// explicit dev/mock env because the dev mock returns payments whose CreatedAt
// is the mock's "now" at seed/replay time, uncorrelated with the aged
// created_at the test rows carry (the keyed-reconcile tests age rows 23h while
// seeding the Square payment at test time, and a retained-key dedup returns
// that seeded payment). The gate mirrors the snapshot-decryption gate, which
// also runs only in a non-dev/mock env.
func replayRevealsNewCharge(r staleRow, pr *square.PaymentResult) (newCharge bool, created time.Time, createdOK bool) {
if IsExplicitDevOrMockEnv() {
return false, time.Time{}, false
}
created, createdOK = parseReplayedCreatedAt(pr)
if !createdOK || r.CreatedAt.IsZero() {
// Cannot prove the replayed payment is the original charge — refuse to
// rescue rather than hide a possible second charge.
return true, created, createdOK
}
return !replayWithinLegitimateWindow(r, pr), created, true
}
// reconcileStalePaymentByKey asks Square for the authoritative status of the
// charge made under a stale pending row's idempotency key and returns the
// tri-state result. The replay sends an IDENTICAL body to the original charge:
// the stored square_request_snapshot (the FULL request — source_id, key,
// amount and every field the original carried), with the source_id overridden
// by the row's CURRENT square_source_id column value (authoritative — a
// same-key reuse refreshes the column while the snapshot JSON is stale).
// Replaying a partial body would
// return IDEMPOTENCY_KEY_REUSED for a RETAINED key and strand the row pending
// forever. Square's idempotency guarantee returns the ORIGINAL payment for a
// retained key (never a second charge); a COMPLETED payment rescues the row to
// 'completed' with its real payment id. A definitive 4xx rejection
// (ErrReplayKeyNotRetained — Square attempted a real charge with the
// expired/used source and refused it, so the charge never happened) or a
// FAILED/CANCELED payment proves the charge never completed and fails the row.
// IDEMPOTENCY_KEY_REUSED is NEVER proof of no charge: with an identical body it
// can only mean the stored source differs from the original (a data bug), so
// the row is left pending with a CRITICAL log. Any OTHER error (transport /
// 5xx / ambiguous) leaves the row pending — the charge may still have completed
// at Square. The second return value is the Square payment id of the completed
// payment ("" otherwise), written back on a rescue.
//
// ENV CONTRACT (finding 4): the sweep and the charge process MUST run with the
// SAME SQUARE_ENVIRONMENT and SQUARE_LOCATION_ID. The stored snapshot embeds
// the location_id used at charge time; if the env/location changes between a
// charge and its replay, the replayed wire body differs and a RETAINED key
// returns IDEMPOTENCY_KEY_REUSED — stranding every retained-key row pending
// (safe) until the 24h blind-fail. This is a single-location deployment; the
// contract is enforced by ops (same env for the sweeper and the API). The
// environment and location are read through square.SquareEnvironment() and
// square.SquareLocationID() — the SAME code path the charge-time HTTP client
// uses (newHTTPClient resolves its base URL and location from those helpers) —
// so the sweep can never drift to a second, independent env read; the
// reconcile below logs the resolved values at each replay as a tripwire.
func reconcileStalePaymentByKey(ctx context.Context, table string, r staleRow) (staleReconcileResult, string) {
snapshot := r.SquareRequestSnapshot
// fallbackBody is true when the row has NO stored square_request_snapshot
// and the minimal body (key + source + amount) is rebuilt. The rebuilt body
// omits reference_id / customer_id / note / buyer_email_address that the
// original charge carried, so for a RETAINED key real Square compares the
// WHOLE body, sees the difference and returns IDEMPOTENCY_KEY_REUSED
// (stranding the row pending) while the dev mock — which compares only the
// source — returns the original payment (rescuing it). Both are money-safe;
// the divergence is dev-vs-prod OBSERVABILITY only, and pre-launch there
// are no prod legacy rows (every charge now stores its snapshot), so the
// fallback semantics are deliberately left unchanged.
fallbackBody := false
if len(bytes.TrimSpace(snapshot)) == 0 {
fallbackBody = true
// Legacy row without a stored request snapshot — rebuild the minimal
// identical body (key + source + amount) exactly as the pre-snapshot
// replay did. Such rows can still be reconciled as long as the stored
// source/amount match the original charge.
fallback, mErr := json.Marshal(square.CreatePaymentReq{
Amount: r.AmountPence,
Currency: "GBP",
SourceID: r.SquareSourceID,
IdempotencyKey: r.IdempotencyKey,
})
if mErr != nil {
log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body for row %s (%v) — leaving pending", table, r.ID, mErr)
return staleReconcileLeavePending, ""
}
snapshot = fallback
}
// PROD-mode snapshot decryption: since the AES-GCM snapshot work, the stored
// square_request_snapshot is ENCRYPTED in a non-mock deployment
// (SQUARE_ENVIRONMENT production/sandbox — the same gate the 2FA
// enforcement uses, IsExplicitDevOrMockEnv); the dev mock stores plaintext.
// Never replay a corrupt snapshot: a decryption failure is a serious error
// that leaves the row pending with a CRITICAL log for manual reconciliation
// instead of replaying garbage (which could return a misleading answer).
if !fallbackBody && !IsExplicitDevOrMockEnv() {
dec, err := decryptSnapshot(snapshot)
if err != nil {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: failed to decrypt the stored request snapshot for row %s (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", table, r.ID, err)
}
snapshot = dec
}
// The stored snapshot embeds the source_id of the ORIGINAL charge, but a
// pending row REUSED by a same-key retry has its square_source_id column
// refreshed to the retry's source while the snapshot JSON stays stale (the
// write side now refreshes the snapshot too). Replaying the stale embedded
// source under a key Square already retains for the new one would return
// IDEMPOTENCY_KEY_REUSED and strand the row pending — so rebuild the replay
// body with the LIVE column value, which is authoritative (set at charge
// time, refreshed on every reuse). The minimal snapshot-less fallback body
// above is already built from the live source and needs no override. An
// unparseable snapshot must never look like proof of no charge — leave the
// row pending for manual reconciliation.
if !fallbackBody && r.SquareSourceID != "" {
var req square.CreatePaymentReq
if err := json.Unmarshal(snapshot, &req); err != nil {
log.Printf("Stale pending %s reconcile by key: failed to parse stored request snapshot for row %s to override the source_id (%v) — leaving pending", table, r.ID, err)
return staleReconcileLeavePending, ""
}
req.SourceID = r.SquareSourceID
overridden, mErr := json.Marshal(req)
if mErr != nil {
log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body with the current square_source_id for row %s (%v) — leaving pending", table, r.ID, mErr)
return staleReconcileLeavePending, ""
}
snapshot = overridden
}
// The replay repeats the stored request snapshot — with the live source_id
// override above — so Square's idempotency dedup returns the original
// payment for a retained key. The
// sweep resolves its Square environment and location through the same
// helpers the charge-time HTTP client uses (square.SquareEnvironment /
// square.SquareLocationID), so the replay and the charge can never read
// two different env sources (see the ENV CONTRACT above). The values are
// logged on each replay as a tripwire for env/location drift between the
// sweeper and the API.
log.Printf("[SWEEP] replay-by-key reconcile for %s row %s: SQUARE_ENVIRONMENT=%q SQUARE_LOCATION_ID=%q (must match the charge-time env/location for identical-body idempotency)", table, r.ID, square.SquareEnvironment(), square.SquareLocationID())
pr, err := SquareClient.ReplayPaymentByKey(ctx, snapshot)
if err != nil {
if errors.Is(err, square.ErrReplayKeyNotRetained) {
// A1: a rejection of a replayed charge against a SAVED-CARD (ccof:)
// source is NOT proof the original charge never happened — the same
// key-retention race that lets the replay land a NEW charge can
// leave that charge even when the probe returns a rejection. Blindly
// failing (and clawing back a till sale's funded gift card) on a
// ccof source could reverse money a replay-created charge already
// took. Leave the row pending and alert ops. A cnon-source
// (spent-nonce) rejection REMAINS definitive proof of no charge (a
// single-use nonce cannot be recharged), keeping the proven-failed
// path and its clawback.
if isSavedCardSource(r.SquareSourceID) {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: Square rejected the identical-body replay (no payment under the stored key) but the row's source is a still-valid saved card (ccof:) — the replay may have landed a NEW charge under an expired idempotency key — leaving row %s PENDING without failing/clawing back — MANUAL RECONCILIATION REQUIRED: verify at Square whether a charge exists before re-issuing", table, r.ID)
}
log.Printf("Stale pending %s reconcile by key: Square has no payment under the stored idempotency key (identical-body replay rejected) — marking failed; the charge provably never happened", table)
return staleReconcileDefinitivelyFailed, ""
}
if square.ErrorCode(err) == "IDEMPOTENCY_KEY_REUSED" {
if fallbackBody {
// Snapshot-less fallback row: the minimal rebuilt body cannot be
// identical to the original charge, so real Square rejects the
// retained-key replay while the dev mock (source-aware only)
// rescues it. The stranded row would otherwise be invisible
// until the 24h blind-fail — surface it now (finding 2).
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED via the minimal snapshot-less fallback body (key=%s) — real Square compares the WHOLE request (reference_id/customer_id/note/buyer_email_address absent from the rebuilt body) and rejects the identical-key replay, stranding the row pending; the dev mock would rescue it — dev-vs-prod observability divergence, NOT proof the charge never happened — MANUAL RECONCILIATION REQUIRED", table, r.IdempotencyKey)
}
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED — the stored square_source_id differs from the original charge's source (data bug); this is NOT proof the charge never happened — leaving pending — MANUAL RECONCILIATION REQUIRED", table)
}
log.Printf("Stale pending %s reconcile by idempotency key hit an ambiguous error (%v) — leaving pending for a later sweep run", table, err)
return staleReconcileLeavePending, ""
}
// Square's terminal payment states are COMPLETED, CANCELED, FAILED;
// APPROVED (authorization-only, delayed capture) and PENDING are
// NON-terminal — both can still transition to COMPLETED, so clawing back
// the funded gift card on either would risk reversing a charge that later
// lands. This app always creates payments with autocomplete (default true),
// so it never produces APPROVED/PENDING rows today, but the classification
// must match Square's documented state machine.
switch pr.Status {
case "COMPLETED":
// A1: a replayed COMPLETED payment created long AFTER the pending row
// is a NEW charge Square made against the still-valid source with an
// expired idempotency key (Square's ~24h key retention is UNVERIFIED —
// square_http_client.go:626), NOT the original charge a retained key
// returns. Rescuing the row with the new payment id would hide the
// second charge behind the original. Leave the row pending and alert
// ops so both charges can be reconciled at Square and the duplicate
// refunded.
if newCharge, created, createdOK := replayRevealsNewCharge(r, pr); newCharge {
lag := "unknown"
if createdOK {
lag = created.Sub(r.CreatedAt).Round(time.Minute).String()
}
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s was created after the pending row %s (lag %s) — a NEW charge under an expired idempotency key (likely a second charge against a still-valid saved card), NOT the original charge — leaving the row PENDING without rescue — MANUAL RECONCILIATION REQUIRED: check Square for both charges and refund the duplicate", table, pr.ID, r.ID, lag)
}
return staleReconcileCompleted, pr.ID
case "CANCELED", "FAILED":
log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed, ""
case "APPROVED", "PENDING":
log.Printf("Stale pending %s is %q at Square (replay by key, non-terminal) — leaving pending for a later sweep run", table, pr.Status)
return staleReconcileLeavePending, ""
default:
log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed, ""
}
}
// leaveGiftCardPurchasePending keeps a gift-card-purchase payment row (payments
// table, booking_id NULL) pending after Square confirms the charge COMPLETED,
// instead of rescuing it to 'completed'. Completing the row would permanently
// block the same-key retry (BuyGiftCard) that reuses the pending record to
// deliver the card — the customer would stay charged with no gift card (C6).
// The row is left pending with a CRITICAL log + admin notification so the retry
// can still deliver the card and an admin is alerted to reconcile manually.
func leaveGiftCardPurchasePending(ctx context.Context, r staleRow) {
log.Printf("CRITICAL: gift card purchase payment %s is COMPLETED at Square but the gift card was never issued (issue transaction failed, retry abandoned) — leaving the payment PENDING so a same-key retry can still deliver the card — MANUAL RECONCILIATION REQUIRED", r.ID)
insertCriticalPaymentNotification(ctx, nil, r.CreatedBy)
}
// insertCriticalPaymentNotification surfaces an unresolved money event in the
// admin notification centre (reason 'critical_payment_log' — the DB-backed
// stand-in for the un-watched CRITICAL payment logs). bookingID is set when the
// issue ties to a booking (untracked terminal charges); userID is set when it
// ties to a user (gift-card purchases). The NOT EXISTS guard keeps ONE
// notification per issue instead of one per sweep run, and requires the prior
// notification to be unacknowledged (acknowledged_at IS NULL) so that after an
// admin acknowledges it, a NEW event for the same booking/user re-notifies —
// matching the webhook copy's guard (handlers/webhooks/square.go) exactly.
func insertCriticalPaymentNotification(ctx context.Context, bookingID, userID *string) {
tag, err := db.Conn.Exec(ctx, `
INSERT INTO admin_notifications (reason, booking_id, user_id, created_at)
SELECT 'critical_payment_log', $1, $2, NOW()
WHERE NOT EXISTS (
SELECT 1 FROM admin_notifications an
WHERE an.reason = 'critical_payment_log'
AND an.booking_id IS NOT DISTINCT FROM $1
AND an.user_id IS NOT DISTINCT FROM $2
AND an.acknowledged_at IS NULL
)
`, bookingID, userID)
if err != nil {
log.Printf("Failed to insert admin_notifications for critical payment issue (booking=%v user=%v): %v", bookingID, userID, err)
return
}
if int(tag.RowsAffected()) > 0 {
log.Printf("Inserted critical-payment admin notification (booking=%v user=%v)", bookingID, userID)
}
}
// notifyStaleRowCritical inserts a critical-payment admin notification for a
// stale row via the shared insertCriticalPaymentNotification helper — the
// notification's deterministic dedup key is the user attribution (the NOT
// EXISTS guard keeps ONE notification per user instead of one per sweep run).
// Payments rows are attributed by the payer (created_by); till_sales rows by
// the funded gift card's redeemed-to user when there is one. booking_id is
// deliberately not used: the sweep runs after the charge process and a
// booking-attributed notification would pin the booking row through the
// admin_notifications FK (no cascade) for as long as the notification survives.
func notifyStaleRowCritical(ctx context.Context, r staleRow) {
var userID *string
if r.CreatedBy != nil {
userID = r.CreatedBy
} else {
userID = r.RedeemToUserID
}
insertCriticalPaymentNotification(ctx, nil, userID)
}
// leavePendingCritical is the shared terminal outcome for a stale pending row
// that cannot be safely resolved by the sweep: it logs a CRITICAL line (the
// caller's existing message), raises the deduped critical-payment admin
// notification via notifyStaleRowCritical, and returns
// staleReconcileLeavePending so the row stays pending for a human. Every
// "do not touch, an operator must reconcile" branch — snapshot-decrypt
// failure, the ccof replay rejection, IDEMPOTENCY_KEY_REUSED and the
// replayed-new-charge case (A4) — now runs the identical log + notification
// outcome through this one helper.
func leavePendingCritical(ctx context.Context, r staleRow, format string, args ...any) (staleReconcileResult, string) {
notifyStaleRowCritical(ctx, r)
log.Printf("CRITICAL: "+format, args...)
return staleReconcileLeavePending, ""
}
// staleReconcileResult is the tri-state outcome of reconciling one stale
// pending row against Square. Only a definitively-resolved outcome touches the
// row: an ambiguous answer (transport error / 5xx) leaves it pending so a
// same-key retry can still reuse it if the charge actually completed.
type staleReconcileResult int
const (
// staleReconcileLeavePending — Square's answer was ambiguous; the row stays
// pending for the next sweep run.
staleReconcileLeavePending staleReconcileResult = iota
// staleReconcileCompleted — Square confirms the charge completed; rescue
// the row to 'completed'.
staleReconcileCompleted
// staleReconcileDefinitivelyFailed — Square proves the charge never
// completed (payment not found / non-completed status); mark the row
// 'failed' exactly as the legacy bulk sweep did.
staleReconcileDefinitivelyFailed
)
// reconcileStalePaymentAtSquare asks Square for the authoritative status of a
// stale pending charge and returns the tri-state result. A COMPLETED payment
// rescues the row to 'completed'; a NOT_FOUND error or any non-COMPLETED status
// proves the charge never completed and fails the row as the legacy bulk sweep
// did. Any OTHER error (transport / 5xx / ambiguous) is NOT treated as a
// definitive failure — the charge may still have completed at Square, and
// marking the row failed would close the double-charge window (blocking a
// same-key retry with a 409) even though the money moved. Such rows stay
// pending for a later run.
func reconcileStalePaymentAtSquare(ctx context.Context, table, squarePaymentID string) staleReconcileResult {
pr, err := SquareClient.GetPayment(ctx, squarePaymentID)
if err != nil {
if squarePaymentErrorIsNotFound(err) {
log.Printf("Stale pending %s reconcile: Square payment %s not found (%v) — marking failed as the legacy sweep would", table, squarePaymentID, err)
return staleReconcileDefinitivelyFailed
}
log.Printf("Stale pending %s reconcile for Square payment %s hit an ambiguous error (%v) — leaving pending for a later sweep run", table, squarePaymentID, err)
return staleReconcileLeavePending
}
// Square's terminal payment states are COMPLETED, CANCELED, FAILED;
// APPROVED (authorization-only, delayed capture) and PENDING are
// NON-terminal — both can still transition to COMPLETED (via
// CompletePayment or delay_action=COMPLETE), so clawing back the funded
// gift card on either would risk reversing a charge that later lands.
// This app always creates payments with autocomplete (default true), so
// it never produces APPROVED/PENDING rows today, but the classification
// must match Square's documented state machine (librarian-verified
// 2026-05-20).
switch pr.Status {
case "COMPLETED":
return staleReconcileCompleted
case "CANCELED", "FAILED":
log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed
case "APPROVED", "PENDING":
log.Printf("Stale pending %s is %q at Square (non-terminal) — leaving pending for a later sweep run", table, pr.Status)
return staleReconcileLeavePending
default:
log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed
}
}
// squarePaymentErrorIsNotFound reports whether a GetPayment error proves the
// payment does not exist at Square. The structured not-found check is primary
// (square.IsNotFound: NOT_FOUND code / HTTP 404 status / plain "HTTP 404" body);
// the message fallback covers only the dev mock's plain "payment not found"
// error, which carries no structured code.
func squarePaymentErrorIsNotFound(err error) bool {
if err == nil {
return false
}
if square.IsNotFound(err) {
return true
}
// Any other structured Square error code is authoritative — never
// substring-match its message.
if square.ErrorCode(err) != "" {
return false
}
msg := strings.ToUpper(err.Error())
return strings.Contains(msg, "NOT_FOUND") ||
strings.Contains(msg, "NOT FOUND")
}
// squareHasCode reports whether err carries a structured Square error code
// equal to any of codes. Errors without a structured code (the dev mock's
// plain errors) match nothing.
func squareHasCode(err error, codes ...string) bool {
if err == nil {
return false
}
code := square.ErrorCode(err)
for _, c := range codes {
if code == c {
return true
}
}
return false
}
// sweepSquarelessManualRefundsAtAttemptCap reconciles Square-less MANUAL refund
// rows that have already hit the attempt cap (maxManualRefundAttempts). The
// Square-less pre-pass inside sweepManualPendingSquareRefunds (refunds.go)
// filters refund_attempts < maxManualRefundAttempts, so a legacy manual refund
// on a payment with no square_payment_id that reached the cap (attempts
// incremented by pre-guard Square attempts) is never reconciled by it and would
// stay 'pending' forever, permanently blocking the over-refund guard (F10).
// Such rows can never be refunded via Square, so they are marked 'failed' and
// surfaced in the admin notification centre for in-person arrangement — the
// same terminal treatment the pre-pass gives rows under the cap. Returns the
// number of rows marked failed.
func sweepSquarelessManualRefundsAtAttemptCap(ctx context.Context) (int, error) {
rows, err := db.Conn.Query(ctx, fmt.Sprintf(`
UPDATE refunds r SET status = 'failed'
FROM payments p
WHERE p.id = r.payment_id
AND r.status = 'pending' AND r.refund_attempts >= %d
AND p.payment_method IN ('online_square', 'in_person_card')
AND p.square_payment_id IS NULL
AND r.origin = 'manual'
RETURNING r.id
`, maxManualRefundAttempts))
if err != nil {
return 0, err
}
defer rows.Close()
var failedIDs []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err == nil {
failedIDs = append(failedIDs, id)
}
}
if err := rows.Err(); err != nil {
return 0, err
}
if len(failedIDs) > 0 {
log.Printf("Marked %d Square-less manual refund(s) at the %d-attempt cap failed (in-person arrangement needed)", len(failedIDs), maxManualRefundAttempts)
}
insertRefundFailedNotifications(ctx, failedIDs)
return len(failedIDs), nil
}
// staleTerminalCheckoutAge is how old a still-pending terminal checkout must
// be before the sweep cancels it. Terminal checkouts normally complete within
// minutes; an hour is far past any legitimate card-reader interaction while
// still short enough that a completed charge can never be misread as stale.
const staleTerminalCheckoutAge = 1 * time.Hour
// SweepStaleTerminalCheckouts cancels terminal (card-machine) checkouts that
// are still PENDING/IN_PROGRESS long after they were created, so the terminal
// stops waiting on a customer who walked away. A checkout created by
// CreateTerminalPayment / CreateTillSale that is never polled would otherwise
// sit live at Square indefinitely; if it later completes it is an invisible,
// untracked charge.
//
// Two tables track live checkout IDs and are both swept:
// - terminal_checkouts: booking terminal checkouts created by
// CreateTerminalPayment. PENDING/IN_PROGRESS rows older than the cutoff
// are resolved at Square first: a checkout still waiting at Square is
// cancelled and re-checked once — if it completed during the cancel window
// the row is marked 'completed' (the poll handler records the payment),
// otherwise 'failed'; a COMPLETED checkout releases the in-flight guard
// (the poll handler records the payment); a definitively
// cancelled/expired checkout is marked 'failed'; an ambiguous status is
// left for a later run.
// - till_sales.square_checkout_id: card-machine till sales. A checkout still
// waiting at Square is cancelled and the sale marked 'failed' (the
// payment_status enum has no 'cancelled' value, and 'failed' is the same
// terminal state the stale-pending sweep uses, blocking the till
// pending-retry path); a checkout that completed during the cancel window
// (or was already COMPLETED when first checked) is RECORDED by the sweep —
// the sale is marked 'completed' with the returned square_payment_id, the
// till mirror of the booking path's recordUntrackedTerminalPayment (a
// never-polled sale would otherwise stay invisible in till reporting until
// the 24h blind-fail).
//
// Each row is checked at Square FIRST and only cancelled when the checkout is
// provably still waiting (ErrCheckoutPending): a COMPLETED checkout is never
// cancelled, and a checkout whose status is unknown (transport error) is left
// alone for a later run. After a cancel the checkout is re-checked once — the
// customer may have completed the payment in the cancel window, in which case
// the row is resolved to 'completed' rather than 'failed'.
func SweepStaleTerminalCheckouts(ctx context.Context) (int, error) {
cutoff := clock.Now().Add(-staleTerminalCheckoutAge)
var pending []staleTerminalCheckoutRow
// Booking terminal checkouts (CreateTerminalPayment) live in the
// terminal_checkouts table. A stale PENDING/IN_PROGRESS row means the
// checkout is still live at Square (or was left after a crash / lost poll).
rows, err := db.Conn.Query(ctx, `
SELECT 'terminal_checkout', checkout_id, checkout_id, booking_id
FROM terminal_checkouts
WHERE status IN ('PENDING', 'IN_PROGRESS')
AND created_at < $1
`, cutoff)
if err != nil {
return 0, err
}
for rows.Next() {
var r staleTerminalCheckoutRow
if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil {
log.Printf("Failed to scan stale terminal checkout row: %v", err)
continue
}
pending = append(pending, r)
}
rows.Close()
// Till-sale card-machine checkouts are tracked on the till_sales row.
rows, err = db.Conn.Query(ctx, `
SELECT 'till_sale', id, square_checkout_id, ''
FROM till_sales
WHERE status = 'pending' AND square_checkout_id IS NOT NULL
AND created_at < $1
`, cutoff)
if err != nil {
return 0, err
}
for rows.Next() {
var r staleTerminalCheckoutRow
if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil {
log.Printf("Failed to scan stale terminal checkout row: %v", err)
continue
}
pending = append(pending, r)
}
rows.Close()
resolved := 0
for _, r := range pending {
// An EMPTY checkout_id is a legacy provisional (pre-Square) row for
// which no Square call was ever possible — resolving it to failed
// directly is safe (R3).
if r.CheckoutID == "" {
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Provisional (pre-Square) terminal checkout row %s (%s) resolved as failed — no live checkout at Square", r.RowID, r.Kind)
continue
}
// A provisional row carries a synthetic "tmp-" checkout_id. It is no
// longer PROVABLY not live (H4): a hard crash between the
// terminal_checkouts insert and the provisional→real UPDATE leaves a
// LIVE checkout at Square (created under the idempotency key embedded
// in the tmp id) while the row still carries the synthetic id.
// Blind-failing would release the in-flight guard while that checkout
// can still complete at the terminal into an untracked charge. Query
// Square first and classify exactly like activeTerminalCheckoutID
// (handlers.go H4): COMPLETED → record the payment and release the
// guard; ErrCheckoutPending → the checkout is still live at Square,
// keep the guard; isTerminalCheckoutError (NOT_FOUND / CANCELED) →
// safe to fail (the crash happened before the Square call, or the
// checkout was cancelled); ambiguous → leave the row in flight. A tmp-
// id that never reached Square (or one Square cannot resolve) comes
// back NOT_FOUND — the expected outcome — and failing it is correct.
if strings.HasPrefix(r.CheckoutID, "tmp-") {
pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case gErr == nil && pr.Status == "COMPLETED":
// The checkout actually completed at the terminal. Record the
// payment if it was never polled/recorded — a COMPLETED charge
// must not stay an invisible untracked charge (H4). Booking
// checkouts record full payment rows; till-sale checkouts
// record the sale row.
if r.Kind == "terminal_checkout" {
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) {
resolved++
}
log.Printf("Provisional terminal checkout %s (%s %s) is COMPLETED at Square — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID)
case errors.Is(gErr, square.ErrCheckoutPending):
// The checkout is still live at Square — keep the in-flight
// guard so a second checkout cannot be created while it can
// still complete.
log.Printf("Provisional terminal checkout %s (%s %s) is still live at Square — leaving pending; the in-flight guard stays held", r.CheckoutID, r.Kind, r.RowID)
case isTerminalCheckoutError(gErr):
// NOT_FOUND (no checkout was ever created — the crash happened
// before the Square call) or CANCELED — safe to resolve failed.
switch {
case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr):
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
log.Printf("Provisional terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID)
case r.Kind == "till_sale":
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("CRITICAL: provisional terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID)
default:
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Provisional (pre-Square) terminal checkout row %q (%s %s) resolved as failed — no live checkout at Square", r.CheckoutID, r.Kind, r.RowID)
}
default:
// Ambiguous error — the checkout's money state at Square is
// unknown. Leave the row in flight rather than releasing the
// guard and allowing a second checkout.
log.Printf("Provisional terminal checkout %s (%s %s) status unknown (%v) — leaving pending for a later sweep", r.CheckoutID, r.Kind, r.RowID, gErr)
}
continue
}
// Conservative status check: only cancel a checkout that is provably
// still waiting at Square. A COMPLETED checkout must never be
// cancelled, and an ambiguous status (network error) is left alone for
// the next run.
pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case errors.Is(gErr, square.ErrCheckoutPending):
// Still live at the terminal — cancel it. A customer can complete
// the payment in the small window between the GetCheckout above and
// the cancel, so re-check once before marking the row failed: a
// COMPLETED charge must never be recorded as failed.
if cErr := SquareClient.CancelCheckout(ctx, r.CheckoutID); cErr != nil {
log.Printf("Failed to cancel stale terminal checkout %s (%s %s): %v", r.CheckoutID, r.Kind, r.RowID, cErr)
continue
}
recheck, rErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case rErr == nil && recheck.Status == "COMPLETED":
// The customer completed the payment during the cancel window.
// Record it if it was never polled/recorded — a COMPLETED
// checkout must not stay an untracked charge (H4). Booking
// checkouts record full payments rows; till-sale checkouts
// record the sale row (finding 3).
if r.Kind == "terminal_checkout" {
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, recheck) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, recheck) {
resolved++
}
log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) but re-check shows COMPLETED — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge)
case isTerminalCheckoutError(rErr) || errors.Is(rErr, square.ErrCheckoutPending):
// The cancel landed (CANCELED / cancel-requested / expired /
// still-reporting-pending-but-now-cancelled) — it can never
// complete, so resolve the row to the terminal 'failed' state.
// A till sale's funded gift card is clawed back (a cancel that
// landed cannot later complete); a booking terminal_checkout
// has no gift card.
if r.Kind == "till_sale" {
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
} else if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) — marked failed", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge)
default:
// The cancel succeeded but the re-check itself is ambiguous —
// leave the row for a later run.
log.Printf("Terminal checkout %s was cancelled but its re-check is ambiguous (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, rErr, r.Kind, r.RowID)
}
case gErr == nil && pr.Status == "COMPLETED":
if r.Kind == "terminal_checkout" {
// The checkout completed at Square but was never polled/recorded
// (the frontend never called GetCheckoutStatus). Record the
// payment rows now — otherwise the charge stays invisible to
// refunds and TotalPaid (H4).
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) {
// A never-polled COMPLETED till-sale checkout must not stay
// pending until the 24h blind-fail (finding 3) — the charge is
// real, so the sale is recorded completed with the payment id.
resolved++
}
case isTerminalCheckoutError(gErr):
// The checkout is cancelled / cancel-requested / expired at Square.
switch {
case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr):
// The error PROVES the checkout can never complete (CANCELED /
// NOT_FOUND, and no bare CANCEL_REQUESTED) — claw back the
// funded gift card along with the failed mark.
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
log.Printf("Terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID)
case r.Kind == "till_sale":
// CANCEL_REQUESTED-only: Square does not promise non-completion
// in that state, so the charge may still land. Mark the sale
// failed but DO NOT claw back the funded gift card.
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("CRITICAL: terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID)
default:
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Terminal checkout %s is definitively terminal (%v) — marked %s %s failed", r.CheckoutID, gErr, r.Kind, r.RowID)
}
default:
// Ambiguous transport/unknown status — leave for a later run.
log.Printf("Terminal checkout %s status unknown (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, gErr, r.Kind, r.RowID)
}
}
return resolved, nil
}
// staleTerminalCheckoutRow is one live-checkout row the sweep reads from
// either table so it can resolve the checkout at Square before touching the
// row. Kind is "terminal_checkout" (booking, terminal_checkouts table) or
// "till_sale" (till_sales.square_checkout_id). BookingID is the booking the
// terminal_checkouts row belongs to ("" for till sales) — needed to record an
// untracked COMPLETED terminal charge (H4).
type staleTerminalCheckoutRow struct {
Kind string
RowID string
CheckoutID string
BookingID string
}
// markTerminalCheckoutRowFailed moves one tracked row to the terminal 'failed'
// state after its checkout is cancelled or proven terminal at Square. Returns
// true when the row was updated (status was still active).
func markTerminalCheckoutRowFailed(ctx context.Context, r staleTerminalCheckoutRow) bool {
var table, where string
if r.Kind == "till_sale" {
table = "till_sales"
where = "id = $1 AND status = 'pending'"
} else {
table = "terminal_checkouts"
where = "checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')"
}
// table is an internal constant ("till_sales"/"terminal_checkouts"), never
// user input, but the identifier is routed through pgx.Identifier.Sanitize
// so no raw, unquoted table name is ever concatenated into the statement.
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE `+where, r.RowID)
if err != nil {
log.Printf("Failed to mark %s %s failed: %v", r.Kind, r.RowID, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// recordUntrackedTerminalPayment records the payments row(s) for a terminal
// checkout that COMPLETED at Square but was never polled/recorded, then marks
// the terminal_checkouts row COMPLETED. The stale sweep otherwise leaves a real
// charge with NO payments row — invisible to refunds and TotalPaid (H4). The
// money-recording work lives in the SHARED recordTerminalPaymentTx core (also
// used by the GetCheckoutStatus poll handler), so both writers run the same
// dedup / FOR UPDATE recheck / insert / M4 split / VAT / guarded checkout
// update; this wrapper supplies the sweep-specific pieces: the advisory lock,
// the cancelled-booking critical admin notification and the CRITICAL completion
// log. Returns true when the checkout row was resolved (payment recorded,
// already recorded, or refused on a cancelled booking); false when recording
// failed (the row is left pending so the next sweep re-runs the whole
// reconcile).
func recordUntrackedTerminalPayment(ctx context.Context, checkoutID, bookingID string, pr *square.PaymentResult) bool {
if pr == nil || pr.SquarePayID == "" {
log.Printf("CRITICAL: terminal checkout %s is COMPLETED at Square but carries no Square payment ID — cannot record the payment — MANUAL RECONCILIATION REQUIRED", checkoutID)
return false
}
// Serialize with the poll handler (GetCheckoutStatus): both record the same
// Square payment, so the advisory lock + dedup SELECT prevent a double
// insert (the idempotency_key UNIQUE constraint is the backstop).
pinConn, err := db.Conn.Acquire(ctx)
if err != nil {
log.Printf("Failed to acquire connection for terminal-completion lock: %v", err)
return false
}
defer pinConn.Release()
lockOK, err := acquireAdvisoryLock(ctx, pinConn, "crussell:terminal:"+pr.SquarePayID)
if err != nil {
log.Printf("Failed to acquire terminal-completion serialization lock for %s: %v", pr.SquarePayID, err)
return false
}
if !lockOK {
log.Printf("Terminal-completion serialization lock for %s not acquired within bound — a poll is already recording this checkout", pr.SquarePayID)
return false
}
defer releasePaymentLock(pinConn, "crussell:terminal:"+pr.SquarePayID)
tx, err := db.Conn.Begin(ctx)
if err != nil {
log.Printf("Failed to begin terminal-completion transaction: %v", err)
return false
}
defer func() {
if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
log.Printf("Failed to rollback terminal-completion transaction: %v", err)
}
}()
paymentID, recErr := recordTerminalPaymentTx(ctx, tx, checkoutID, bookingID, pr)
if recErr != nil {
if errors.Is(recErr, errTerminalBookingNotPayable) {
// Sweep-only behaviour: the poll surfaces a 409 to a live caller,
// but a sweep has no one to tell, so alert ops in the admin
// notification centre that money was taken at Square and MUST be
// refunded manually (the core already committed the checkout's
// 'failed' mark).
insertCriticalPaymentNotification(ctx, &bookingID, nil)
return true
}
// The core logged the failure; the row is left pending so the next
// sweep run re-runs the whole reconcile.
return false
}
log.Printf("CRITICAL: recorded untracked terminal charge %s (booking %s) from stale checkout %s — payment row %s created (never polled by the frontend)", pr.SquarePayID, bookingID, checkoutID, paymentID)
return true
}
// errTerminalBookingNotPayable is returned by recordTerminalPaymentTx when the
// booking moved out of a payable state (cancelled / lapsed / no-show) between
// the terminal charge completing at Square and the record attempt. The core has
// ALREADY committed the terminal_checkouts 'failed' mark (and a non-payable
// commit failure is wrapped with this sentinel too), so the caller must not
// roll back: the poll surfaces a 409 conflict, the sweep inserts the critical
// admin notification and resolves the row.
var errTerminalBookingNotPayable = errors.New("booking is no longer payable for a terminal payment")
// recordTerminalPaymentTx records a COMPLETED terminal checkout as payments
// rows, applying the M4 tip split, per-record VAT and booking completion. Both
// the GetCheckoutStatus poll handler and the stale-terminal sweep call it so
// the money-recording logic exists exactly once.
//
// CALL-SITE CONTRACT: both call sites must keep passing the SAME inputs — the
// caller's transaction, the terminal_checkouts checkout_id, the booking id and
// the *square.PaymentResult returned by GetCheckout — and each must acquire the
// advisory lock on "crussell:terminal:<SquarePayID>" on its own pinned pool
// connection BEFORE calling (the core runs inside the caller's transaction).
// The core commits the transaction and completes a now-fully-paid booking; the
// caller owns only the error mapping (errTerminalBookingNotPayable vs generic),
// the HTTP response / sweep return value and any post-commit behaviour (e.g.
// the sweep's critical notification).
func recordTerminalPaymentTx(ctx context.Context, tx pgx.Tx, checkoutID, bookingID string, pr *square.PaymentResult) (string, error) {
service := NewPaymentService()
// Dedup by Square payment ID: a concurrent poll (or a prior sweep run)
// already recorded this charge — release the in-flight guard and return the
// existing row so the caller reports the same payment id. The checkout is
// marked COMPLETED under the same guarded status predicate used everywhere
// else, so a row already resolved to 'failed' is never resurrected.
var existingID string
if err := tx.QueryRow(ctx, `
SELECT id FROM payments
WHERE booking_id = $1 AND square_payment_id = $2
`, bookingID, pr.SquarePayID).Scan(&existingID); err == nil {
if _, upErr := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); upErr != nil {
slog.Error("Failed to mark terminal checkout completed on dedup", "checkout_id", checkoutID, "err", upErr)
return "", fmt.Errorf("mark terminal checkout %s completed: %w", checkoutID, upErr)
}
if cErr := tx.Commit(ctx); cErr != nil {
slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", cErr)
return "", cErr
}
return existingID, nil
} else if !errors.Is(err, pgx.ErrNoRows) {
slog.Error("Failed to check for existing terminal payment", "checkout_id", checkoutID, "booking_id", bookingID, "square_payment_id", pr.SquarePayID, "err", err)
return "", err
}
// Re-check the booking status under the advisory lock (mirrors
// GetCheckoutStatus): a cancellation/eviction that committed between the
// terminal charge completing at Square and this record attempt must not
// produce a completed payment on a cancelled/lapsed/no-show booking — the
// cancellation refund path computes refunds from completed payments and
// would silently exclude this charge. Mark the checkout failed and alert
// ops: money was taken at Square and MUST be refunded manually.
var recheckStatus string
// FOR UPDATE (C5): serializes against the cancellation path's lock on the
// same row so a concurrent cancellation cannot commit between this recheck
// and the transaction commit below.
if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, bookingID).Scan(&recheckStatus); err != nil {
slog.Error("CRITICAL: Square payment was processed but re-reading booking status failed — manual reconciliation required", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "err", err)
return "", err
}
if !bookingStatusAllowsCompletedPayment(recheckStatus) {
slog.Error("CRITICAL: Square payment was processed but booking is no longer payable — marking checkout failed; money taken at Square MUST be refunded manually", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "status", recheckStatus)
if _, upErr := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'failed', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); upErr != nil {
slog.Error("CRITICAL: Square payment landed on a non-payable booking but marking checkout failed errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", upErr)
}
if cErr := tx.Commit(ctx); cErr != nil {
slog.Error("CRITICAL: Square payment landed on a non-payable booking and committing the checkout-failed mark errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", cErr)
// Keep the sentinel wrapped so BOTH callers still surface the
// conflict path (poll 409 / sweep notification) even when the
// failed-mark commit itself failed.
return "", fmt.Errorf("%w: %v", errTerminalBookingNotPayable, cErr)
}
return "", errTerminalBookingNotPayable
}
// The payment type the admin charged is recorded on the checkout row by
// CreateTerminalPayment; fall back to 'full' for legacy rows.
var checkoutPaymentType string
if err := tx.QueryRow(ctx, `
SELECT payment_type FROM terminal_checkouts WHERE checkout_id = $1
`, checkoutID).Scan(&checkoutPaymentType); err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
slog.Error("Failed to read payment type for checkout", "checkout_id", checkoutID, "err", err)
}
checkoutPaymentType = "full"
}
idempotencyKey := bookingID + "-terminal-" + strconv.FormatInt(pr.Amount, 10) + "-" + pr.SquarePayID
record := PaymentRecord{
BookingID: bookingID,
PaymentType: checkoutPaymentType,
PaymentMethod: "in_person_card",
Status: "completed",
Amount: float64(pr.Amount) / 100.0,
SquarePaymentID: &pr.SquarePayID,
IdempotencyKey: &idempotencyKey,
Fees: float64(pr.Fees) / 100.0,
CreatedAt: clock.Now(),
UpdatedAt: clock.Now(),
}
// M4 split: a terminal charge above the remaining booking value is a tip —
// record it as its own record so only the booking portion is refundable.
var records []PaymentRecord
bookingInfo, bErr := service.GetBookingPaymentInfo(ctx, bookingID)
if bErr == nil && bookingInfo != nil {
charged := float64(pr.Amount) / 100.0
remainingBookingValue := math.Max(0, bookingInfo.TotalAmount-bookingInfo.TotalPaid)
bookingPortion := math.Min(charged, remainingBookingValue)
bookingPortion = math.Round(bookingPortion*100) / 100
tipAmount := math.Round((charged-bookingPortion)*100) / 100
if tipAmount > 0.004 {
records = buildTerminalSplitRecords(record, bookingInfo, bookingPortion, tipAmount)
}
}
if len(records) == 0 {
records = []PaymentRecord{record}
}
primary := records[0]
paymentID, err := service.CreatePaymentRecordTx(ctx, tx, primary, nil)
if err != nil {
slog.Error("Failed to create payment record for terminal charge", "square_payment_id", pr.SquarePayID, "err", err)
return "", err
}
ApplyVATToBookingPayment(ctx, tx, paymentID)
for _, rec := range records[1:] {
pid, cErr := service.CreatePaymentRecordTx(ctx, tx, rec, nil)
if cErr != nil {
slog.Error("Failed to create terminal tip split record", "square_payment_id", pr.SquarePayID, "err", cErr)
return "", cErr
}
ApplyVATToBookingPayment(ctx, tx, pid)
}
// Release the in-flight guard: this checkout is now recorded.
if _, err := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); err != nil {
slog.Error("Failed to mark terminal checkout completed", "checkout_id", checkoutID, "err", err)
return "", err
}
if err := tx.Commit(ctx); err != nil {
slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", err)
return "", err
}
// The booking may now be fully paid — complete it like the poll handler does.
completeFullyPaidBooking(ctx, bookingID)
return paymentID, nil
}
// recordUntrackedTillSalePayment records a stale card-machine till sale whose
// checkout COMPLETED at Square but was never polled/recorded: the sale is
// marked 'completed' with the returned square_payment_id written back. A
// never-polled COMPLETED checkout would otherwise leave the sale pending until
// the 24h blind-fail — the charge is real but invisible in till reporting
// (money-safe: the gift card was funded pre-charge, a blind-fail never claws
// back, and failed rows reject retries — but the till reporting would be wrong,
// finding 3). There is no split/deposit concept for till sales — a single
// completed row, exactly the update the till poll handler (GetTillCheckoutStatus)
// performs. The WHERE status = 'pending' guard makes it safe against a
// concurrent poll: only one of them wins the update, the loser sees 0 rows and
// backs off. Returns true when the sale was updated; false when the checkout
// carries no Square payment id (the sale is left PENDING — never blind-failed
// — with a CRITICAL log for manual reconciliation) or the sale was already
// resolved by someone else.
func recordUntrackedTillSalePayment(ctx context.Context, saleID string, pr *square.PaymentResult) bool {
if pr == nil || pr.SquarePayID == "" {
log.Printf("CRITICAL: terminal checkout is COMPLETED at Square but carries no Square payment ID — cannot record till sale %s — leaving it PENDING — MANUAL RECONCILIATION REQUIRED", saleID)
return false
}
tag, err := db.Conn.Exec(ctx, `
UPDATE till_sales SET status = 'completed', square_payment_id = $1, updated_at = NOW()
WHERE id = $2 AND status = 'pending'
`, pr.SquarePayID, saleID)
if err != nil {
log.Printf("Failed to record untracked terminal till sale %s as completed: %v", saleID, err)
return false
}
if int(tag.RowsAffected()) == 0 {
// A concurrent poll (or a prior sweep run) already recorded the sale —
// nothing left for this run to do.
log.Printf("Terminal till sale %s was already resolved — skipping untracked terminal completion record", saleID)
return false
}
log.Printf("CRITICAL: recorded untracked terminal till-sale charge %s (sale %s) from a stale checkout — sale marked completed (never polled by the frontend)", pr.SquarePayID, saleID)
return true
}
// isTerminalCheckoutError reports whether a GetCheckout error proves the
// checkout can never complete. Square's HTTP client returns ErrCheckoutPending
// for a still-live checkout and surfaces a definitively CANCELED status as a
// "square: checkout <id> is CANCELED (not COMPLETED)" error; an expired
// checkout returns a structured NOT_FOUND API error (the mock uses a plain
// "checkout not found"). Any other error (timeout, 5xx) leaves the money state
// ambiguous, so the checkout must stay in flight.
//
// Structured codes are authoritative when present: NOT_FOUND (via
// square.IsNotFound) and an explicit CANCELED / CANCEL_REQUESTED code classify
// as terminal. The formatted-message match applies only to errors that carry
// no structured code — the dev mock's plain errors and the real client's
// client-side "is <status> (not COMPLETED)" error, which it synthesizes
// without a squareAPIError.
func isTerminalCheckoutError(err error) bool {
if err == nil || errors.Is(err, square.ErrCheckoutPending) {
return false
}
if square.IsNotFound(err) || squareHasCode(err, "CANCELED", "CANCEL_REQUESTED") {
return true
}
// Any other structured Square error code is authoritative — never
// substring-match its message.
if square.ErrorCode(err) != "" {
return false
}
msg := strings.ToUpper(err.Error())
return strings.Contains(msg, "CANCELED") ||
strings.Contains(msg, "CANCEL_REQUESTED") ||
strings.Contains(msg, "NOT_FOUND") ||
strings.Contains(msg, "NOT FOUND")
}
// isCheckoutDefinitivelyDead reports whether a GetCheckout error PROVES the
// checkout can never complete — the condition under which a till sale's funded
// gift card may be clawed back. It is stricter than isTerminalCheckoutError:
// a CANCEL_REQUESTED-only classification (Square does not promise non-completion
// in that state) is NOT definitive proof, so the charge may still land and the
// funding must stay put. Only an explicit CANCELED or NOT_FOUND status is
// definitive; a CANCEL_REQUESTED message counts only when it ALSO carries
// CANCELED.
//
// Structured codes are authoritative when present: CANCELED and NOT_FOUND are
// definitive, a bare CANCEL_REQUESTED code is not. The formatted-message match
// applies only to errors that carry no structured code (dev mock plain errors,
// and the real client's client-side "is <status> (not COMPLETED)" error).
func isCheckoutDefinitivelyDead(err error) bool {
if err == nil || errors.Is(err, square.ErrCheckoutPending) {
return false
}
if code := square.ErrorCode(err); code != "" {
return code == "CANCELED" || code == "NOT_FOUND"
}
// Non-structured errors: an HTTP 404 / plain 404 body is definitive
// (IsNotFound); the message match covers the client-side CANCELED status
// error and the mock's plain not-found errors.
if square.IsNotFound(err) {
return true
}
msg := strings.ToUpper(err.Error())
if !strings.Contains(msg, "CANCELED") && !strings.Contains(msg, "NOT_FOUND") && !strings.Contains(msg, "NOT FOUND") {
return false
}
if strings.Contains(msg, "CANCEL_REQUESTED") && !strings.Contains(msg, "CANCELED") {
return false
}
return true
}
// clawBackTillSaleFunding reverts the gift-card funding of a till sale whose
// card-machine checkout is provably dead, marking the sale failed at the same
// time. The terminal-checkout sweep's rows carry no gift-card context, so the
// sale's item/amount/redeem target are looked up here (is_create via the
// created_at equality with the gift card) and handed to the claim-first
// revertGiftCardFunding. A sale with no gift card (future retail product /
// orphaned item) is only marked failed. Returns true when the sale was
// resolved to failed; false when it was already resolved by someone else, had
// no gift card, or the clawback failed (CRITICAL logged).
func clawBackTillSaleFunding(ctx context.Context, tillSaleID string) bool {
var itemType string
var itemID sql.NullString
var totalAmount float64
var redeemedBy sql.NullString
var isCreate *bool
err := db.Conn.QueryRow(ctx, `
SELECT ts.item_type, ts.item_id, ts.total_amount, gc.redeemed_by,
(ts.created_at = gc.created_at) AS is_create
FROM till_sales ts
LEFT JOIN gift_cards gc ON gc.id = ts.item_id
WHERE ts.id = $1
`, tillSaleID).Scan(&itemType, &itemID, &totalAmount, &redeemedBy, &isCreate)
if err != nil {
log.Printf("CRITICAL: failed to look up till sale %s for funding clawback: %v — MANUAL RECONCILIATION REQUIRED", tillSaleID, err)
return false
}
if itemType != "gift_card" || !itemID.Valid || itemID.String == "" || isCreate == nil {
// No gift card to claw back — mark the sale failed without touching
// any card.
tag, upErr := db.Conn.Exec(ctx, `
UPDATE till_sales SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, tillSaleID)
if upErr != nil {
log.Printf("Failed to mark till sale %s failed: %v", tillSaleID, upErr)
return false
}
return int(tag.RowsAffected()) > 0
}
action := "topup"
if *isCreate {
action = "create"
}
var redeem *string
if redeemedBy.Valid && redeemedBy.String != "" {
redeem = &redeemedBy.String
}
if revErr := revertGiftCardFunding(ctx, action, itemID.String, totalAmount, redeem, tillSaleID); revErr != nil {
if errors.Is(revErr, errTillSaleNotPending) {
log.Printf("Till sale %s was already resolved (not pending) — skipping funding clawback", tillSaleID)
return false
}
log.Printf("CRITICAL: funding clawback for till sale %s (gift card %s) failed: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", tillSaleID, itemID.String, revErr)
return false
}
return true
}