package payments import ( "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" "log" "log/slog" "math" "strconv" "strings" "time" "crussell/clock" "crussell/db" "crussell/internal/adminnotify" "crussell/internal/square" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" ) // SweepStalePendingPayments resolves pending payment records that are older // than Square's idempotency-key retention window (~24h). A pending record // means the DB committed but the Square charge outcome is unknown; it // normally resolves on a same-key client retry. But if the client abandoned // the attempt, the record stays pending forever — and retrying it after the // key expires would ISSUE A SECOND CHARGE (Square no longer dedups). Failing // stale pendings closes that double-charge window: a late retry finds a // 'failed' record and stops instead of charging again. // // Before failing a row, the sweep reconciles it against Square: a // genuinely-charged row (Square success, DB post-charge failure) with a // square_payment_id is rescued to 'completed' instead of being swept to // 'failed' with no automatic resolution — the money would otherwise be lost // in limbo (MINOR-R3). Reconciliation is deliberately minimal: status + // updated_at only, no split/VAT recomputation (that is the handler's job; the // row is >24h stale and this is a reconciliation rescue). A row with NO // square_payment_id but a STORED idempotency key (the lost-response case — the // charge may have completed at Square with the response never received) is // reconciled at Square by replaying the key (ReplayPaymentByKey) at an EARLIER // 22h cutoff, while the key is still inside Square's ~24h retention window: a // COMPLETED charge is rescued to 'completed' with the real square_payment_id // written back, and a charge Square proves never happened is failed. Rows with // neither a square_payment_id nor a stored key cannot be reconciled and are // failed with a WARN exactly as the legacy sweep did. // // Only online/till card payments can be pending — cash/giftcard/on_the_house // are committed synchronously and never enter this state. Both the payments // table and till_sales carry pending card-sale rows and are swept here. // // CLOCK-SOURCE CONTRACT (M3): EVERY age-guard cutoff in this package is // computed from clock.Now() (the single wall-clock source) — never from a DB // NOW()-derived comparison. The DB's NOW() is used only at WRITE time (the // created_at/updated_at rows this sweep reads); the sweep's cutoffs are // computed in Go from clock.Now() and passed into the SQL as parameters, so a // Go-side comparison can never mix a DB-computed cutoff with a Go-computed // one. This matters at the 23h/24h boundary: Square's idempotency-key // retention (~24h) is compared against the DB-stored created_at, and using one // consistent source prevents the decision from flipping on the skew between // two independent clocks. const stalePendingPaymentAge = 24 * time.Hour // stalePendingKeyedAge is how old a pending row with a stored idempotency key // (but no square_payment_id — the lost-response case) must be before the sweep // reconciles it at Square by replaying the key. It is deliberately 2h EARLIER // than stalePendingPaymentAge so the replay lands comfortably inside Square's // ~24h idempotency-key retention window: replaying at exactly 24h risks the key // already having expired, and an expired key would make the probe rejection // look like "never charged" even when the charge actually landed. Rows older // than stalePendingPaymentAge when swept can no longer be replayed // trustworthily and fall back to the legacy blind-fail + WARN. const stalePendingKeyedAge = 22 * time.Hour // SweepKeyedReplayAge returns the age at which a keyed pending row (stored // idempotency key, no square_payment_id) becomes eligible for the sweep's // replay-by-key reconcile. Exported for the webhooks package so the // payment.completed orphan-replay detection can gate on whether a pending // origin row could actually have been replayed by the sweep: a pending row // younger than this age can never be a sweep-minted duplicate's origin, so it // must not be marked failed by a webhook that raced the charge response. func SweepKeyedReplayAge() time.Duration { return stalePendingKeyedAge } // b1DuplicateRefundAttemptCap caps how many times the sweep may auto-refund a // replay-induced duplicate charge (B1) under one stale pending row's expired // idempotency key. A REJECTED refund writes NO refunds row, so the in-flight // guard (hasInFlightSweepDuplicateRefund) stays false and the next run would // re-replay the SAME expired key — Square mints ANOTHER charge, the refund is // rejected again, and the loop stacks unauthorized charges over the row's 24h // life. The cap stops the loop: a row at the cap is failed with a CRITICAL // notification and its key is never replayed. Mirrors maxManualRefundAttempts // (refunds.go). const b1DuplicateRefundAttemptCap = 3 // SweepStalePendingPayments resolves stale pending payments and till sales; see the rationale block on stalePendingPaymentAge above. func SweepStalePendingPayments(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-stalePendingPaymentAge) keyedCutoff := clock.Now().Add(-stalePendingKeyedAge) // Pass 0: stranded Square-less MANUAL refund rows at the attempt cap // (maxManualRefundAttempts). The Square-less pre-pass inside // sweepManualPendingSquareRefunds (refunds.go) reconciles only rows with // refund_attempts < maxManualRefundAttempts; legacy rows // that already hit the cap are never reconciled by it and would stay // 'pending' forever, permanently blocking the over-refund guard. Such rows // can never be refunded via Square (no square_payment_id), so they are // marked 'failed' + admin-notified here for in-person arrangement (F10). sqlessRefundCount, sqlessErr := sweepSquarelessManualRefundsAtAttemptCap(ctx) if sqlessErr != nil { log.Printf("Failed to reconcile Square-less manual refunds at the 3-attempt cap: %v", sqlessErr) } // Pass 1 (earlier cutoff): rows with a stored idempotency key but NO // square_payment_id are the lost-response case — the charge may have landed // at Square with the response lost. They are reconciled at Square by // replaying the key (ReplayPaymentByKey) while the key is still inside // Square's ~24h retention window: a COMPLETED charge is rescued to // 'completed' with the real square_payment_id, a charge Square proves never // happened is failed (and a till sale's funded gift card clawed back), an // ambiguous answer leaves the row pending for the next run, and a row // already past the retention window is blind-failed with a WARN exactly as // the legacy sweep did (replaying an expired key would misread the probe // rejection as "never charged"). payKeyedCount, payKeyedCompleted, payKeyedUnverified, err := sweepKeyedStaleRows(ctx, "payments", keyedCutoff) if err != nil { return 0, err } tillKeyedCount, tillKeyedCompleted, tillKeyedUnverified, err := sweepKeyedStaleRows(ctx, "till_sales", keyedCutoff) if err != nil { return 0, err } // Pass 2 (legacy 24h cutoff): rows WITH a square_payment_id are reconciled // by payment id; rows with neither a payment id nor a stored key cannot be // reconciled and are failed directly (WARN — the charge outcome is unknown). payCount, payCompleted, err := sweepStaleRows(ctx, "payments", cutoff) if err != nil { return 0, err } tillCount, tillCompleted, err := sweepStaleRows(ctx, "till_sales", cutoff) if err != nil { return 0, err } total := sqlessRefundCount + payKeyedCount + tillKeyedCount + payCount + tillCount payTotal := payKeyedCount + payCount tillTotal := tillKeyedCount + tillCount completed := payKeyedCompleted + tillKeyedCompleted + payCompleted + tillCompleted if total > 0 { log.Printf("[SWEEP] Resolved %d stale pending payments (%d payments, %d till sales) older than %s — late retries will be rejected, preventing a second Square charge; %d reconciled to completed against Square (%d payments, %d till sales)", total, payTotal, tillTotal, stalePendingPaymentAge, completed, payKeyedCompleted+payCompleted, tillKeyedCompleted+tillCompleted) } // Routine sweep bookkeeping, not an incident: failing stale pending rows is // the sweep's DESIGNED behaviour. A row swept to failed may still have been // charged at Square with a lost response, so note it at WARN level for an // admin doing a periodic money reconciliation — but this fires on every // normal run and must not be elevated to CRITICAL (which is reserved for // genuinely unrecoverable post-charge branches). Rows the keyed reconcile // PROVED never charged are deliberately excluded from the WARN (they cannot // have moved money); only rows failed without a reconcile proof — keyless // rows and keyed rows past the retention window — are counted. if payKeyedUnverified > 0 { log.Printf("[SWEEP] WARN: %d pending payments with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response — verify before refunding/charging", payKeyedUnverified) } if tillKeyedUnverified > 0 { log.Printf("[SWEEP] WARN: %d pending till sales with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response", tillKeyedUnverified) } if payCount > 0 { log.Printf("[SWEEP] WARN: %d pending payments marked failed may have been charged at Square with a lost response — verify before refunding/charging", payCount-payCompleted) } if tillCount > 0 { log.Printf("[SWEEP] WARN: %d pending till sales marked failed may have been charged at Square with a lost response", tillCount-tillCompleted) } return total, nil } // staleRow is one stale pending row read by the sweep so it can reconcile // rows that carry a Square reference BEFORE failing them. For till_sales rows // the gift-card context needed to claw back funded money on a definitive // failure is carried alongside: the card id, who it was redeemed to, whether // this sale created the card (created_at equality — provable because both // timestamps are the transaction-start NOW() in the same tx), and the sale // amount (the exact funding this sale added). type staleRow struct { ID string SquarePaymentID string // IdempotencyKey is the deterministic Square idempotency key stored on the // pending row ("" when absent). Rows WITH a key but NO square_payment_id are // the lost-response case: the sweep replays the key at Square to learn the // true charge outcome before declaring failure. IdempotencyKey string // SquareSourceID is the row's CURRENT square_source_id — the source_id sent // in the latest CreatePayment attempt, refreshed whenever a same-key retry // reuses the pending row. The replay-by-key overrides the stored snapshot's // embedded source with this value so the replayed body matches the source // the retained key actually used. Replaying a different body would return // IDEMPOTENCY_KEY_REUSED, which proves nothing about whether the charge // landed. SquareSourceID string // SquareRequestSnapshot is the verbatim original CreatePayment request JSON // stored on the row at charge time (square_request_snapshot) — the FULL // body the replay-by-key must repeat (source, key, amount, customer_id, // reference_id, note, buyer_email_address, verification_token, ...). // Square's idempotency dedup compares the whole request, so a replay built // from partial row data returns IDEMPOTENCY_KEY_REUSED for a RETAINED key // and the row stays pending forever (safe but never auto-rescued). Nil for // legacy rows — the sweep then rebuilds the minimal body from the stored // key + source + amount. SquareRequestSnapshot []byte // AmountPence is the row's charge amount in pence — the amount the original // CreatePayment used. The replay-by-key must repeat it so Square's // idempotency dedup returns the original payment. AmountPence int64 // CreatedAt is the pending row's creation time; rows already past Square's // idempotency-key retention window cannot be replayed trustworthily. CreatedAt time.Time // BookingID is the payments row's booking_id ("" when the row has none). // A payments row with NO booking is a gift-card purchase (BuyGiftCard // inserts without a booking): rescuing it to 'completed' on a Square // COMPLETED reconcile would permanently block the same-key retry that // delivers the card, so such rows are kept pending instead (C6). BookingID *string // CreatedBy is the payments row's created_by user id (gift-card purchases // always carry the purchaser), used to attribute the critical-payment admin // notification. CreatedBy *string ItemID string // till_sales.item_id — the gift card ("" when NULL / non-gift-card) RedeemToUserID *string // gift_cards.redeemed_by — user credited by a create-with-redeem IsCreate bool // true when this sale created the gift card (timestamps equal) HasGiftCard bool // false when the LEFT JOIN found no gift_cards row (gc.id IS NULL) TotalAmount float64 // till_sales.total_amount — the funding this sale added // B1Attempts counts how many times the sweep has auto-refunded a // replay-induced duplicate charge under this row's expired idempotency key. // A REJECTED refund (or the cap) means the key must never be replayed again. B1Attempts int } // acquireTillSaleSweepLock serializes the sweep's fail/clawback of a stale // till_sale on the SAME advisory lock a same-key retry pins while its Square // charge is mid-flight: "crussell:till:" (till.go — the retry // path holds it from before the Square round-trip until the row is completed). // // The race this closes (FIX 1, MAJOR): without the lock, the sweep can probe // Square for the charge under the key while the retry's charge has not landed // yet, read "no payment under the key", mark the sale failed and claw back the // funded gift card — then the retry's charge lands. The retry's completion // UPDATE (guarded on status='pending') matches 0 rows and logs CRITICAL: the // customer is charged at Square AND the funding is clawed back. By holding the // same lock before failing/clawing-back, the sweep either serializes after the // retry (the row is already 'completed' — the fail/clawback is a 0-row no-op) // or the lock is contended and the row is DEFERRED to a later pass (the retry // may still land its charge). // // Returns (release, true) when the lock is held — the caller MUST call // release() before resolving the row; (nil, false) when the lock could not be // acquired within the bounded try-lock (~3s) — a same-key retry is likely // mid-flight at Square, so the caller must SKIP the row this pass (it is // reconciled again next sweep) — or a pinned pool connection could not be // acquired. A row with NO idempotency key returns a no-op release with true: // no retry path can hold a lock on it, so there is nothing to serialize on. func acquireTillSaleSweepLock(ctx context.Context, r staleRow) (release func(), ok bool) { if r.IdempotencyKey == "" { return func() {}, true } pinConn, err := db.Conn.Acquire(ctx) if err != nil { log.Printf("Failed to acquire connection for till-sale sweep lock on stale row %s: %v", r.ID, err) return nil, false } lockKey := "crussell:till:" + r.IdempotencyKey acquired, lErr := acquireAdvisoryLock(ctx, pinConn, lockKey) if lErr != nil { log.Printf("Failed to acquire till-sale sweep lock %s for stale row %s: %v", lockKey, r.ID, lErr) pinConn.Release() return nil, false } if !acquired { // A same-key retry holds the lock and its Square charge may be // mid-flight — deferring avoids failing/clawing-back a sale whose // charge can still land. The row is reconciled again next pass. log.Printf("Stale pending till_sale %s: till-sale lock %s is contended within the bound — a same-key retry may be mid-flight at Square — deferring the row to a later sweep pass", r.ID, lockKey) pinConn.Release() return nil, false } return func() { releasePaymentLock(pinConn, lockKey) pinConn.Release() }, true } // tillSalePendingAfterLock re-reads a till_sales row's CURRENT stored // idempotency_key and status AFTER the sweep acquired its advisory lock and // reports whether the fail/clawback may still proceed. // // This closes the residual window of the FIX-1 lock (BUG 2, MAJOR): a same-key // retry of a KEYLESS sale locks the derived BASE key (crussell:till:) // while its slot scan may resolve a SUFFIXED final key (base-2) that becomes // the STORED key — the key the sweep locks (till.go). Until the retry side // pins the final key too, the sweep and the retry do NOT serialize, so the // retry can complete the sale (status → 'completed', its Square charge // landed) while the sweep waited on — or just acquired — its lock on the stale // stored key. A blind fail/clawback in that window would reverse the funding // of a sale whose charge is real. Re-reading the row under the lock detects // the retry's completion; a row that is no longer 'pending' — or whose stored // key changed while the sweep waited — is left alone and reconciled again // under its current key next sweep. func tillSalePendingAfterLock(ctx context.Context, r staleRow) bool { var key, status string if err := db.Conn.QueryRow(ctx, ` SELECT COALESCE(idempotency_key, ''), status FROM till_sales WHERE id = $1 `, r.ID).Scan(&key, &status); err != nil { log.Printf("Failed to re-read till sale %s after acquiring its sweep lock (%v) — skipping the fail/clawback this pass; the row is reconciled next sweep", r.ID, err) return false } if status != "pending" { log.Printf("Stale pending till sale %s is now %q (a same-key retry completed or failed it while the sweep waited) — skipping the fail/clawback; the row is reconciled next sweep", r.ID, status) return false } if key != r.IdempotencyKey { log.Printf("Stale pending till sale %s's stored idempotency key changed from %q to %q while the sweep waited (a same-key retry re-scanned its slot) — skipping the fail/clawback on the stale key; the row is reconciled next sweep under its current key", r.ID, r.IdempotencyKey, key) return false } return true } // sweepStaleRows resolves the stale pending rows of one table. Rows with a // square_payment_id are reconciled at Square first (COMPLETED → 'completed', // anything else → 'failed' exactly as the legacy bulk UPDATE did); rows // without one cannot be reconciled and are failed directly. A till_sales row // whose reconcile PROVES the charge never completed (NOT_FOUND / non-COMPLETED) // also claws back the funded gift card atomically with the failed mark; the // blind-fail path (no square_payment_id — the charge may have landed) never // claws back. Returns the total rows resolved and how many were rescued to // 'completed'. func sweepStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved int, completed int, err error) { switch table { case "payments", "till_sales": default: return 0, 0, fmt.Errorf("sweep: unknown stale table %q", table) } stale, err := fetchStaleRows(ctx, table, cutoff, false) if err != nil { return 0, 0, err } for _, r := range stale { // B1: a row whose auto-refund of a replay-induced duplicate charge is // still PENDING at Square must never be blind-failed (or have its gift // card clawed back) — the refund is non-terminal and may still settle. // The B1 re-poll pass resolves it on settlement. if hasInFlightSweepDuplicateRefund(ctx, table, r.ID) { log.Printf("Stale pending %s row %s has a sweep auto-refund of a duplicate charge still pending at Square — leaving pending until the refund settles", table, r.ID) continue } if r.SquarePaymentID != "" { switch reconcileStalePaymentAtSquare(ctx, table, r.SquarePaymentID) { case staleReconcileCompleted: if table == "payments" && r.BookingID == nil { // Gift-card purchase row: the charge landed at Square but // the gift card was never delivered (C6). Completing the row // permanently blocks the same-key retry that delivers the // card — leave it pending and alert. leaveGiftCardPurchasePending(ctx, r) continue } // F3: a charge Square reports COMPLETED must never be // completed on a booking that was cancelled during the pending // window — the cancellation refund path computes refunds from // completed payments and would miss it, charging the customer // with NO automatic refund. Re-read bookings.status (FOR // UPDATE) and refuse on a cancelled booking. Rows with no // booking (gift-card purchases, till_sales) are never gated. if table == "payments" { switch gateStalePaymentRescueOnBooking(ctx, r, r.SquarePaymentID) { case staleBookingGateRefused: resolved++ continue case staleBookingGateUnknown: continue } } if rescueStaleRowCompleted(ctx, table, r) { resolved++ completed++ } continue case staleReconcileLeavePending: // Square's answer was ambiguous (transport/5xx) — the charge // may still be in flight at Square. Do NOT touch the row: the // next sweep run reconciles it again, and a same-key retry // must still be able to reuse the pending row if the charge // actually completed. log.Printf("Stale pending %s row %s left pending (Square reconcile ambiguous) — will retry next sweep", table, r.ID) continue } // staleReconcileDefinitivelyFailed falls through to the fail path. } // Fail path. A till-sale reconcile that PROVED the charge never // completed (NOT_FOUND / non-COMPLETED) claws back the funded gift // card atomically with the failed mark (HIGH-1). The blind-fail path // (no square_payment_id — the charge outcome is unknown) NEVER claws // back: the money may have landed at Square. FIX 1: a till_sale is // resolved under its own till-sale advisory lock so the sweep cannot // race a late same-key retry whose Square charge is mid-flight. if table == "till_sales" { release, lockOK := acquireTillSaleSweepLock(ctx, r) if !lockOK { continue } // BUG 2: the lock may not serialize against a keyless-sale retry // (its lock is the derived base key, ours is the stored suffix key) // — re-read the row and refuse to fail/clawback a sale a retry // already completed or re-keyed while the sweep waited. if !tillSalePendingAfterLock(ctx, r) { release() continue } if r.SquarePaymentID != "" && r.HasGiftCard { if clawbackTillSaleFunding(ctx, r) { resolved++ } } else if failStaleRow(ctx, table, r.ID) { resolved++ // A5a: blind-failing a row with NO square_payment_id leaves the // charge outcome unknown (the money may have landed at Square // with a lost response), so surface it in the admin notification // centre. Rows that reach this line WITH a square_payment_id // were PROVED never-charged by the by-id reconcile and need no // alert. if r.SquarePaymentID == "" { notifyStaleRowCritical(ctx, r) } } release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ // A5a: blind-failing a row with NO square_payment_id leaves the // charge outcome unknown (the money may have landed at Square with a // lost response), so surface it in the admin notification centre. // Rows that reach this line WITH a square_payment_id were PROVED // never-charged by the by-id reconcile and need no alert. if r.SquarePaymentID == "" { notifyStaleRowCritical(ctx, r) } } } return resolved, completed, nil } // sweepKeyedStaleRows resolves stale pending rows that carry a stored // idempotency key but no square_payment_id — the lost-response case, where the // charge may have completed at Square with the response never reaching the // app. Such rows are reconciled at Square by replaying the key while it is // still inside Square's ~24h retention window (the earlier stalePendingKeyedAge // cutoff guarantees this): // // - a COMPLETED payment under the key → rescued to 'completed' with the real // square_payment_id written back (the customer WAS charged); // - a payment Square proves never happened (unknown key / FAILED status) → // marked 'failed', and a till sale's funded gift card is clawed back // (reconcile PROVED the funding has no charge behind it); // - an ambiguous answer (transport/5xx) → left pending for the next run; // - a row already older than the retention window when swept → cannot be // replayed trustworthily (an expired key would misread the probe rejection // as "never charged" even when the charge landed), so it is blind-failed // with a WARN exactly as the legacy sweep did. // // Returns the total rows resolved, how many were rescued to 'completed', and // how many were marked failed WITHOUT a reconcile proof (blind-fail WARNs). func sweepKeyedStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved, completed, unverifiable int, err error) { switch table { case "payments", "till_sales": default: return 0, 0, 0, fmt.Errorf("sweep: unknown stale table %q", table) } stale, err := fetchStaleRows(ctx, table, cutoff, true) if err != nil { return 0, 0, 0, err } // Square's idempotency-key retention window closes at stalePendingPaymentAge; // a row older than that when swept can no longer be replayed trustworthily. replayExpired := clock.Now().Add(-stalePendingPaymentAge) for _, r := range stale { // B1: a row whose auto-refund of a replay-induced duplicate charge is // still PENDING at Square must be left alone — re-playing its expired // idempotency key can land ANOTHER charge, and blind-failing or clawing // back while the refund is non-terminal would reverse money that may // still come back. The B1 re-poll pass resolves it on settlement. if hasInFlightSweepDuplicateRefund(ctx, table, r.ID) { log.Printf("Stale pending %s row %s has a sweep auto-refund of a duplicate charge still pending at Square — leaving pending until the refund settles", table, r.ID) continue } // B1 (CRITICAL-HIGH): never re-replay a key whose B1 auto-refund of a // replay-induced duplicate charge has already FAILED. hasFailedSweepDuplicateRefund // catches a refund Square rejected AFTER the sweep accepted it pending — // the FAILED-webhook demotion clears the in-flight guard above, so without // this check the next run would re-replay the expired key and mint ANOTHER // charge. The b1_attempts cap bounds a refund that keeps failing without // ever writing a refunds row (REJECTED at call time) or keeps erroring // (transport) — the current row count is re-read from the DB each run. // A row here is failed (NOT clawed back — the duplicate money may stand // at Square) with a CRITICAL notification for manual reconciliation. if hasFailedSweepDuplicateRefund(ctx, table, r.ID) || r.B1Attempts >= b1DuplicateRefundAttemptCap { // FIX 1: a till_sale is failed under its own till-sale advisory // lock so a same-key retry mid-flight at Square cannot land its // charge onto a row the sweep just marked failed. if table == "till_sales" { release, lockOK := acquireTillSaleSweepLock(ctx, r) if !lockOK { continue } if !tillSalePendingAfterLock(ctx, r) { release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ } notifyStaleRowCritical(ctx, r) log.Printf("Stale pending %s row %s has a FAILED or attempt-capped B1 auto-refund of a replay-induced duplicate charge (b1_attempts=%d) — never re-replaying the expired key; marked failed — MANUAL RECONCILIATION REQUIRED: check Square for the duplicate charge and refund it", table, r.ID, r.B1Attempts) release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ } notifyStaleRowCritical(ctx, r) log.Printf("Stale pending %s row %s has a FAILED or attempt-capped B1 auto-refund of a replay-induced duplicate charge (b1_attempts=%d) — never re-replaying the expired key; marked failed — MANUAL RECONCILIATION REQUIRED: check Square for the duplicate charge and refund it", table, r.ID, r.B1Attempts) continue } if r.CreatedAt.Before(replayExpired) { // Key retention window already closed — replaying would misread an // expired key as "never charged". Blind-fail + WARN exactly as the // legacy sweep did; a till sale's funded gift card is NOT clawed // back (the charge outcome is unknown, the money may have landed). // A5a: this blind-fail can be hiding a real charge (the lost // response the stored key was meant to reconcile), so the admin // notification centre must surface it too. // FIX 1: a till_sale is blind-failed under its own till-sale // advisory lock so a same-key retry mid-flight at Square cannot // land its charge onto a row the sweep just marked failed. if table == "till_sales" { release, lockOK := acquireTillSaleSweepLock(ctx, r) if !lockOK { continue } if !tillSalePendingAfterLock(ctx, r) { release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ unverifiable++ notifyStaleRowCritical(ctx, r) } release() log.Printf("Stale pending %s row %s has a stored idempotency key but is already past Square's key retention window — marked failed without a replay reconcile (may have been charged with a lost response)", table, r.ID) continue } if failStaleRow(ctx, table, r.ID) { resolved++ unverifiable++ notifyStaleRowCritical(ctx, r) } log.Printf("Stale pending %s row %s has a stored idempotency key but is already past Square's key retention window — marked failed without a replay reconcile (may have been charged with a lost response)", table, r.ID) continue } switch res, sqPayID := reconcileStalePaymentByKey(ctx, table, r); res { case staleReconcileCompleted: if table == "payments" && r.BookingID == nil { // Gift-card purchase row (C6): the charge landed at Square but // the card was never delivered. Completing the row blocks the // same-key retry that delivers the card — leave it pending. leaveGiftCardPurchasePending(ctx, r) continue } // F3: same money-safety gate as the by-id rescue above — a charge // Square reports COMPLETED must never be completed on a booking // that was cancelled during the pending window. if table == "payments" { switch gateStalePaymentRescueOnBooking(ctx, r, sqPayID) { case staleBookingGateRefused: resolved++ continue case staleBookingGateUnknown: continue } } if rescueKeyedStaleRowCompleted(ctx, table, r, sqPayID) { resolved++ completed++ } case staleReconcileLeavePending: // Ambiguous replay — the charge may still be in flight. Leave the // row pending; the next sweep run reconciles it again. log.Printf("Stale pending %s row %s left pending (replay-by-key reconcile ambiguous) — will retry next sweep", table, r.ID) case staleReconcileDefinitivelyFailed: // Square proved the charge never happened (key unknown / FAILED) — // a till sale's funded gift card is clawed back atomically with the // failed mark, unlike the blind-fail path where the outcome is unknown. // FIX 1: the fail/clawback runs under the row's till-sale advisory // lock so the sweep cannot race a late same-key retry whose Square // charge is mid-flight (its "no payment under the key" probe would // be premature). if table == "till_sales" { release, lockOK := acquireTillSaleSweepLock(ctx, r) if !lockOK { continue } if !tillSalePendingAfterLock(ctx, r) { release() continue } if r.HasGiftCard { if clawbackTillSaleFunding(ctx, r) { resolved++ } } else if failStaleRow(ctx, table, r.ID) { resolved++ } release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ } case staleReconcileDefinitivelyFailedNoClawback: // B1 (CRITICAL-HIGH): the auto-refund of the replay-induced // duplicate charge was definitively REJECTED (or errored) at Square — // the duplicate charge stands and the money is REAL at Square, so a // till sale's funded gift card is NOT clawed back. Mark the parent // row failed, raise the CRITICAL notification, and set b1_attempts // to the cap so the expired key is never replayed (each replay // would mint ANOTHER charge). FIX 1: a till_sale is failed under // its own till-sale advisory lock so a same-key retry mid-flight at // Square cannot land its charge onto a row the sweep just marked // failed. if table == "till_sales" { release, lockOK := acquireTillSaleSweepLock(ctx, r) if !lockOK { continue } if !tillSalePendingAfterLock(ctx, r) { release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ } notifyStaleRowCritical(ctx, r) setB1AttemptsToCap(ctx, table, r.ID) // Loop-B finding (MED): a capped-fail on a till_sale leaves its // funded gift card outstanding — the duplicate charge at Square is // REAL and a manual Square refund is the only reversal. Surface the // outstanding funding with a gift_card_transactions trace so the // operator sees it (see insertOutstandingGiftCardFundingTrace for // the manual-resolution path). if r.HasGiftCard { insertOutstandingGiftCardFundingTrace(ctx, r) } release() continue } if failStaleRow(ctx, table, r.ID) { resolved++ } notifyStaleRowCritical(ctx, r) setB1AttemptsToCap(ctx, table, r.ID) // Loop-B finding (MED): a capped-fail on a till_sale leaves its // funded gift card outstanding — the duplicate charge at Square is // REAL and a manual Square refund is the only reversal. Surface the // outstanding funding with a gift_card_transactions trace so the // operator sees it (see insertOutstandingGiftCardFundingTrace for // the manual-resolution path). if table == "till_sales" && r.HasGiftCard { insertOutstandingGiftCardFundingTrace(ctx, r) } } } return resolved, completed, unverifiable, nil } // fetchStaleRows reads the stale pending rows of one table that are older than // cutoff. keyedOnly restricts the query to rows that can be reconciled by // replaying a stored idempotency key: those with a key but no square_payment_id // (rows WITH a square_payment_id are reconciled by payment id in the main // pass). The legacy till_sales sweep only touches card methods — cash and // on_the_house are committed synchronously and never pending, but the // predicate is kept so behaviour is byte-identical for any unexpected row. func fetchStaleRows(ctx context.Context, table string, cutoff time.Time, keyedOnly bool) ([]staleRow, error) { keyedPredicate := "" if keyedOnly { keyedPredicate = ` AND idempotency_key IS NOT NULL AND square_payment_id IS NULL` } methodFilter := "" if table == "till_sales" { methodFilter = ` AND payment_method IN ('online_square', 'in_person_card')` } var rows pgx.Rows var err error if table == "till_sales" { rows, err = db.Conn.Query(ctx, ` SELECT ts.id, COALESCE(ts.square_payment_id, ''), COALESCE(ts.idempotency_key, ''), COALESCE(ts.square_source_id, ''), COALESCE(ts.square_request_snapshot, ''), ts.created_at, ts.item_id, gc.redeemed_by, (ts.created_at = gc.created_at) AS is_create, (gc.id IS NOT NULL) AS has_gift_card, ts.total_amount, ts.b1_attempts FROM till_sales ts LEFT JOIN gift_cards gc ON gc.id = ts.item_id WHERE ts.status = 'pending' AND ts.created_at < $1`+methodFilter+keyedPredicate+` `, cutoff) } else { // table is an internal constant ("payments"), never user input, but the // identifier is routed through pgx.Identifier.Sanitize — the same // treatment failStaleRow / rescueStaleRowCompleted give the table name — // so no raw, unquoted table name is ever concatenated into the statement. rows, err = db.Conn.Query(ctx, ` SELECT id, COALESCE(square_payment_id, ''), COALESCE(idempotency_key, ''), COALESCE(square_source_id, ''), COALESCE(square_request_snapshot, ''), created_at, amount, booking_id, created_by, b1_attempts FROM `+pgx.Identifier{table}.Sanitize()+` WHERE status = 'pending' AND created_at < $1`+keyedPredicate+` `, cutoff) } if err != nil { return nil, err } defer rows.Close() var stale []staleRow for rows.Next() { r, scanErr := scanStaleRow(table, rows) if scanErr != nil { log.Printf("Failed to scan stale pending row from %s: %v", table, scanErr) continue } stale = append(stale, r) } return stale, rows.Err() } // scanStaleRow scans one row produced by fetchStaleRows into a staleRow, // deriving the replay amount (in pence) from the stored pound figure — the // original CreatePayment amount the replay-by-key must repeat. func scanStaleRow(table string, rows pgx.Rows) (staleRow, error) { var r staleRow if table == "till_sales" { var itemID, redeemedBy, snapshot sql.NullString var isCreate *bool var hasGiftCard bool if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &itemID, &redeemedBy, &isCreate, &hasGiftCard, &r.TotalAmount, &r.B1Attempts); err != nil { return r, err } r.SquareRequestSnapshot = []byte(snapshot.String) r.ItemID = itemID.String if redeemedBy.Valid && redeemedBy.String != "" { r.RedeemToUserID = &redeemedBy.String } r.IsCreate = isCreate != nil && *isCreate r.HasGiftCard = hasGiftCard r.AmountPence = int64(math.Round(r.TotalAmount * 100)) // NOTE: float64→int64 pence conversion can drift by ±1p between charge and replay. // A full fix would store pence as int64 throughout (out of scope for this change). return r, nil } var amount float64 var bookingID, createdBy, snapshot sql.NullString if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &amount, &bookingID, &createdBy, &r.B1Attempts); err != nil { return r, err } r.SquareRequestSnapshot = []byte(snapshot.String) r.AmountPence = int64(math.Round(amount * 100)) // NOTE: float64→int64 pence conversion can drift by ±1p between charge and replay. // A full fix would store pence as int64 throughout (out of scope for this change). if bookingID.Valid && bookingID.String != "" { b := bookingID.String r.BookingID = &b } if createdBy.Valid && createdBy.String != "" { c := createdBy.String r.CreatedBy = &c } return r, nil } // failStaleRow marks one stale pending row 'failed'. Returns true when the row // was updated (status was still 'pending'). func failStaleRow(ctx context.Context, table, id string) bool { // table is an internal constant ("payments"/"till_sales"), never user // input, but the identifier is routed through pgx.Identifier.Sanitize so no // raw, unquoted table name is ever concatenated into the statement. tag, err := db.Conn.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, id) if err != nil { log.Printf("Failed to mark stale pending row %s failed: %v", id, err) return false } return int(tag.RowsAffected()) > 0 } // rescueStaleRowCompleted marks one stale pending row (whose square_payment_id // already resolves to a COMPLETED charge) 'completed'. For a payments row with // a booking this mirrors the live-path recording semantics (MEDIUM-3): the // row's charge is re-split through buildSplitRecords so any overflow beyond // the booking's remaining obligation is carved out as its own // payment_type='tip' record (never mis-booked as service revenue), and the // fully-paid completion check runs so a rescue that settles the booking // triggers the completion side-effects exactly like the live path. Returns // true when the row was updated. func rescueStaleRowCompleted(ctx context.Context, table string, r staleRow) bool { return rescueStaleRowCompletedTx(ctx, table, r.ID, "", r) } // rescueKeyedStaleRowCompleted marks a keyed stale pending row 'completed' with // the square_payment_id returned by the replay — the lost-response rescue — and // applies the same split/completion semantics as the by-id rescue above // (MEDIUM-3). Returns true when the row was updated. func rescueKeyedStaleRowCompleted(ctx context.Context, table string, r staleRow, squarePaymentID string) bool { return rescueStaleRowCompletedTx(ctx, table, r.ID, squarePaymentID, r) } // rescueStaleRowCompletedTx is the shared implementation behind both rescue // paths. The split records are computed BEFORE the status flip (while the row // is still 'pending', so GetBookingPaymentInfo's paid ledger excludes it — // exactly where the live path computes buildSplitRecords), and the flip + // split insert + completion run in ONE transaction so a crash can never leave // the row completed but the carve missing. func rescueStaleRowCompletedTx(ctx context.Context, table, id, squarePaymentID string, r staleRow) bool { tx, err := db.Conn.Begin(ctx) if err != nil { log.Printf("Failed to begin rescue transaction for stale pending %s row %s: %v", table, id, err) return false } defer func() { if rErr := tx.Rollback(ctx); rErr != nil && !errors.Is(rErr, pgx.ErrTxClosed) { log.Printf("Failed to rollback rescue transaction for stale pending %s row %s: %v", table, id, rErr) } }() // Pre-compute the split while the row is still 'pending' so the booking's // paid ledger excludes it (mirroring the live post-charge path, where // buildSplitRecords runs before the primary row is marked completed). records := buildStaleRescueRecords(ctx, tx, r, squarePaymentID) var tag pgconn.CommandTag if squarePaymentID != "" { tag, err = tx.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', square_payment_id = $1, updated_at = NOW() WHERE id = $2 AND status = 'pending' `, squarePaymentID, id) } else { tag, err = tx.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, id) } if err != nil { log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err) return false } if int(tag.RowsAffected()) == 0 { // Already resolved concurrently — nothing to do. if cErr := tx.Commit(ctx); cErr != nil { log.Printf("Failed to commit rescue no-op for stale pending %s row %s: %v", table, id, cErr) } return false } if len(records) > 0 { applyStaleRescueRecords(ctx, tx, r, records) } if table == "till_sales" { // M5: the synchronous till-completion path (GetTillCheckoutStatus) // applies VAT via ApplyVATToTillSale; a sweep rescue must do the same or // the rescued sale silently drops out of VAT reporting. The SQL function // is idempotent (guarded on vat_amount IS NULL) inside the same tx. ApplyVATToTillSale(ctx, tx, r.ID) } if cErr := tx.Commit(ctx); cErr != nil { log.Printf("Failed to commit rescue of stale pending %s row %s: %v", table, id, cErr) return false } return true } // buildStaleRescueRecords computes the payment records a rescued stale pending // booking payment should be split into (MEDIUM-3). It mirrors the live // post-charge path: buildSplitRecords partitions the row's charge into the // deposit/balance/full booking portion plus a carved payment_type='tip' record // for any overflow beyond the booking's remaining obligation — so a rescue can // never mis-book the overflow as service revenue. The row MUST still be // 'pending' when this runs (the paid ledger must exclude it). Rows that need // no split (tip-type rows, rows without a booking, till_sales) return nil. // Any error is logged and nil is returned: the row is still rescued (status // flip) un-split — never worse than the pre-fix minimal flip — and surfaces in // the log for manual reconciliation. func buildStaleRescueRecords(ctx context.Context, tx pgx.Tx, r staleRow, squarePaymentID string) []PaymentRecord { if r.BookingID == nil { return nil } var paymentType, paymentMethod string var createdBy sql.NullString if err := tx.QueryRow(ctx, `SELECT payment_type, payment_method, created_by FROM payments WHERE id = $1 FOR UPDATE`, r.ID).Scan(&paymentType, &paymentMethod, &createdBy); err != nil { log.Printf("MEDIUM-3: failed to read payment row %s for rescue split (booking %s): %v — the row is completed un-split; manual reconciliation recommended", r.ID, *r.BookingID, err) return nil } if paymentType == "tip" { // A tip row is already tip-type — no carve is needed (a tip can never // fully pay a booking, so the completion check would be a no-op too). return nil } info, err := NewPaymentService().GetBookingPaymentInfo(ctx, *r.BookingID) if err != nil || info == nil { log.Printf("MEDIUM-3: failed to load booking info for rescue split of payment %s (booking %s): %v — the row is completed un-split; manual reconciliation recommended", r.ID, *r.BookingID, err) return nil } amount := float64(r.AmountPence) / 100.0 spID := r.SquarePaymentID if squarePaymentID != "" { spID = squarePaymentID } var createdByPtr *string if createdBy.Valid && createdBy.String != "" { c := createdBy.String createdByPtr = &c } primary := PaymentRecord{ BookingID: *r.BookingID, PaymentType: paymentType, PaymentMethod: paymentMethod, Status: "completed", Amount: amount, SquarePaymentID: &spID, CreatedBy: createdByPtr, CreatedAt: clock.Now(), UpdatedAt: clock.Now(), } if r.IdempotencyKey != "" { k := r.IdempotencyKey primary.IdempotencyKey = &k } records, err := buildSplitRecords(primary, paymentType, info, amount) if err != nil { log.Printf("MEDIUM-3: buildSplitRecords rejected the rescue split for payment %s (booking %s): %v — the row is completed un-split; manual reconciliation recommended", r.ID, *r.BookingID, err) return nil } return records } // applyStaleRescueRecords applies the pre-computed split records inside the // rescue transaction: the primary row is aligned to records[0] and the // remaining split records (balance/full portion + any carved tip) are inserted // with their derived idempotency keys. The fully-paid completion check then // runs so a rescue that settles the booking triggers the completion // side-effects (loyalty stamps, campaigns, deposits_required) the live path // would have run. Errors are logged with the row left as-is for manual // reconciliation — the money was already completed at Square, so the rescue // must never abort on a bookkeeping failure. func applyStaleRescueRecords(ctx context.Context, tx pgx.Tx, r staleRow, records []PaymentRecord) { if len(records) == 0 { return } // The align UPDATE clears the VAT fields exactly like the live post-charge // path (handlers.go align UPDATE sets is_vat_applicable = FALSE, vat_rate = // NULL, vat_amount = NULL, net_amount = NULL before re-applying VAT). // WITHOUT the clearing (pre-fix bug) the pending row's VAT — computed at // step-1 insert time on the FULL pre-split charge (handlers.go step-1 // ApplyVATToBookingPayment) — survived the align, and the re-apply below was // a silent no-op because apply_vat_to_payment is guarded on vat_amount IS // NULL (init-script.sql): the rescued primary row kept VAT on the wrong // (larger) base. Clearing first makes the recompute effective on the split // amount, and the SQL's payment_type != 'tip' guard leaves an all-tip // rescue (the primary aligned to the carved tip record) VAT-free. if _, upErr := tx.Exec(ctx, ` UPDATE payments SET amount = $1, payment_type = $2, fees = $3, is_vat_applicable = FALSE, vat_rate = NULL, vat_amount = NULL, net_amount = NULL, updated_at = NOW() WHERE id = $4 `, records[0].Amount, records[0].PaymentType, records[0].Fees, r.ID); upErr != nil { log.Printf("MEDIUM-3: failed to align rescued payment %s to its split primary (%v) — manual reconciliation recommended", r.ID, upErr) return } // M5: the synchronous post-charge path applies VAT per record // (ApplyVATToBookingPayment); the sweep re-applies it after the align — // now effective because the align cleared the stale VAT fields above. // apply_vat_to_payment is idempotent (guarded on vat_amount IS NULL) and // skips discount/on_the_house/tip rows internally. ApplyVATToBookingPayment(ctx, tx, r.ID) svc := NewPaymentService() for _, rec := range records[1:] { pid, cErr := svc.CreatePaymentRecordTx(ctx, tx, rec, nil) if cErr != nil { log.Printf("MEDIUM-3: failed to insert rescue split record for payment %s (%v) — manual reconciliation recommended", r.ID, cErr) return } ApplyVATToBookingPayment(ctx, tx, pid) } if bookingIsFullyPaid(ctx, tx, *r.BookingID) { completeActiveBookingFromPayment(ctx, tx, *r.BookingID) } } // staleBookingGate is the outcome of the pre-completion booking-status recheck // for a stale pending payment row whose charge Square reports COMPLETED. type staleBookingGate int const ( // staleBookingGateAllowed — the booking is still payable; the rescue may proceed. staleBookingGateAllowed staleBookingGate = iota // staleBookingGateRefused — the booking is cancelled/lapsed/no-show; the row // was marked FAILED and a critical admin notification raised. staleBookingGateRefused // staleBookingGateUnknown — the booking status could not be read; the row was // left pending (never completed on an unknown state). staleBookingGateUnknown ) // gateStalePaymentRescueOnBooking re-reads the booking of a stale pending // payment row before the sweep rescues it to 'completed' on a Square COMPLETED // reconcile. It mirrors the live-path guard bookingStatusAllowsCompletedPayment // (handlers.go): a charge that lands AFTER the booking was cancelled must never // be silently completed — the cancellation refund path computes refunds from // completed payments and would miss it, charging a customer for a cancelled // booking with NO automatic refund (F3). // // The booking status is re-read FOR UPDATE inside a transaction so the // decision serializes against a concurrent cancellation (C5 pattern, mirrors // recordTerminalPaymentTx). On a cancelled/lapsed/no-show booking the payment // row is marked FAILED (not completed) and — M2 — a pending cancellation // refund row is AUTO-CREATED for the full stranded charge (mirroring // ProcessCancellationRefundTx's record shape and origin) so the pending-refund // sweep (SweepPendingSquareRefunds → processChargeGroup) issues the Square // refund automatically: previously the charge was only failed + admin-notified // and the customer stayed charged with NO automatic refund row. The square // payment id (authoritative — from the by-id reconcile, or the replay result // on the keyed path) is written onto the row when it is missing, because the // refund sweep refunds by square_payment_id. A critical admin notification is // still raised so the operator sees the auto-refunded stranded charge. A // booking whose status cannot be read is never completed — the row is left // pending for the next sweep run. Rows with no booking (till_sales; gift-card // purchases are diverted by the callers before this helper) are never gated. func gateStalePaymentRescueOnBooking(ctx context.Context, r staleRow, squarePaymentID string) staleBookingGate { if r.BookingID == nil { return staleBookingGateAllowed } tx, err := db.Conn.Begin(ctx) if err != nil { log.Printf("CRITICAL: failed to begin booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err) return staleBookingGateUnknown } defer func() { if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { log.Printf("Failed to rollback booking-status recheck for stale pending payment %s: %v", r.ID, err) } }() var status string if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, *r.BookingID).Scan(&status); err != nil { // Booking gone or unreadable: the money state is unknown. Never // complete on an unknown state — a completed payment on a vanished // booking would strand the charge outside the refund system. log.Printf("CRITICAL: Square reports stale pending payment %s COMPLETED but re-reading booking %s status failed (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err) return staleBookingGateUnknown } if bookingStatusAllowsCompletedPayment(status) { // Payable — release the booking lock; the caller rescues the row. if cErr := tx.Commit(ctx); cErr != nil { log.Printf("CRITICAL: failed to commit booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, cErr) return staleBookingGateUnknown } return staleBookingGateAllowed } // Cancelled / lapsed / no-show booking: completing the payment would // charge a customer for a booking the cancellation flow already closed, // with NO automatic refund. Mark the row FAILED (not completed) so the // same-key retry can never reuse it, and (M2) auto-create the refund row // for the full stranded charge. tag, upErr := tx.Exec(ctx, ` UPDATE `+pgx.Identifier{"payments"}.Sanitize()+` SET status = 'failed', square_payment_id = COALESCE(square_payment_id, $2), updated_at = NOW() WHERE id = $1 AND status = 'pending' `, r.ID, squarePaymentID) if upErr != nil { log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — marking the row failed errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, upErr) return staleBookingGateUnknown } // M2: a refund row for the full stranded charge. Same shape and origin // ('cancellation') ProcessCancellationRefundTx records, so the pending // Square refund sweep (SweepPendingSquareRefunds) aggregates and issues it // at Square. The deterministic key mirrors the cancellation path // (paymentID + "-square-" + amount pence) and the UNIQUE conflict guard // makes a re-run idempotent. created_by falls back to the row's payer. if int(tag.RowsAffected()) > 0 && r.AmountPence > 0 { if _, rfErr := tx.Exec(ctx, ` INSERT INTO refunds (payment_id, booking_id, amount, status, reason, idempotency_key, created_by, created_at, origin) VALUES ($1, $2, $3, 'pending', $4, $5, $6, NOW(), 'cancellation') ON CONFLICT (idempotency_key) DO NOTHING `, r.ID, *r.BookingID, float64(r.AmountPence)/100.0, "Stranded charge on cancelled booking "+status+" — auto-refunded by stale-pending sweep", r.ID+"-square-"+strconv.FormatInt(r.AmountPence, 10), r.CreatedBy); rfErr != nil { log.Printf("CRITICAL: stale pending payment %s (booking %s) is COMPLETED but creating its auto-refund row errored (%v) — MANUAL RECONCILIATION REQUIRED: refund the charge manually", r.ID, *r.BookingID, rfErr) } } if cErr := tx.Commit(ctx); cErr != nil { log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — committing the failed mark errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, cErr) return staleBookingGateUnknown } if int(tag.RowsAffected()) > 0 { insertCriticalPaymentNotification(ctx, r.BookingID, r.CreatedBy) log.Printf("CRITICAL: stale pending payment %s is COMPLETED at Square but booking %s is %q — payment marked FAILED instead of completed; an automatic refund row was created for the stranded charge — verify the Square refund settles", r.ID, *r.BookingID, status) return staleBookingGateRefused } // The row was already resolved concurrently — nothing left to refuse. return staleBookingGateUnknown } // clawbackTillSaleFunding claws back a stale pending till sale's funded gift // card (atomically with the failed mark) after a reconcile PROVED the charge // never completed — the funding has no charge behind it. A sale with no gift // card is only marked failed. Returns true when the sale was resolved to // failed; false when it was already resolved by someone else, had no gift card, // or the clawback failed (CRITICAL logged). func clawbackTillSaleFunding(ctx context.Context, r staleRow) bool { action := "topup" if r.IsCreate { action = "create" } if revErr := revertGiftCardFunding(ctx, action, r.ItemID, r.TotalAmount, r.RedeemToUserID, r.ID); revErr != nil { if errors.Is(revErr, errTillSaleNotPending) { log.Printf("Stale pending till sale %s was already resolved (not pending) — skipping funding clawback", r.ID) return false } log.Printf("CRITICAL: failed to claw back gift card %s funding for stale till sale %s: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", r.ItemID, r.ID, revErr) return false } return true } // insertOutstandingGiftCardFundingTrace surfaces a till sale's gift-card funding // that remains outstanding after a B1 capped-fail (the auto-refund of the // replay-induced duplicate charge was REJECTED/errored — the duplicate charge // stands at Square, so the funding is NOT auto-clawed back: reversing money the // merchant still holds would lose it). The trace row lets the operator see the // outstanding funding and drives the manual-resolution path: // // 1. refund the duplicate charge at Square FIRST (it is real money — the // customer never authorized it), // 2. then run clawbackTillSaleFunding (revertGiftCardFunding with the sale's // action, gift_card_id and amount) to reverse the funding atomically once // the duplicate is gone. func insertOutstandingGiftCardFundingTrace(ctx context.Context, r staleRow) { if _, err := db.Conn.Exec(ctx, ` INSERT INTO gift_card_transactions (gift_card_id, transaction_type, amount, reference_type, reference_id, user_id, notes) VALUES ($1, 'awaiting_reversal', $2, 'till_sale', $3, $4, $5) `, r.ItemID, r.TotalAmount, r.ID, r.RedeemToUserID, "B1 capped-fail: duplicate charge stands at Square — refund it there FIRST, then run the till-sale funding clawback manually"); err != nil { log.Printf("Failed to insert outstanding gift-card funding trace for till sale %s: %v", r.ID, err) } } // replayRescueLowerBoundSkew is the lower-bound tolerance for a replayed // COMPLETED payment to still be treated as the ORIGINAL charge under a retained // key rather than a provably-new duplicate. Rows are inserted pending-first, so // natural order is row.CreatedAt < Square's created_at by ~0.5-1s — and the DB // clock can run AHEAD of Square's (independent NTP drift, VM pause/resume), // which makes a retained-key dedup return the ORIGINAL charge with created < // r.CreatedAt. Without this tolerance the sweep would declare that original // payment a "new charge" and auto-refund a charge the customer legitimately // authorized. A payment created within the skew BEFORE the row is therefore the // ORIGINAL and is rescued (Loop B MEDIUM — a >1min-ahead DB clock previously // stranded such a charge pending until the 24h blind-fail); only a payment // created more than the skew before the row, or after the sweep's replay // instant (replayAt), is ambiguous / a provable expired-key duplicate. const replayRescueLowerBoundSkew = 5 * time.Minute // replayRescueUpperBoundSkew is the tolerance applied to the UPPER bound of // the legitimate window (replayWithinLegitimateWindow): a replayed COMPLETED // payment created this far AFTER the sweep's own replay instant (replayAt) is // still treated as the ORIGINAL charge / a legit same-key retry rather than a // provably-new expired-key duplicate. The strict bound (created.Before(replayAt)) // had no clock-skew tolerance: Square's clock can run a few seconds AHEAD of // the app's, so a legitimately-authorized same-key retry that raced the sweep // (the retry charge lands at Square a moment before the sweep's replay while // Square's clock already reads past replayAt) would be misclassified as the // sweep's OWN replay-created duplicate and AUTO-REFUNDED — reversing a charge // the customer authorized. The 5s window absorbs that skew. The trade-off — // the sweep's own replay-created charge could be created within 5s of replayAt // by the same skew and be rescued rather than auto-refunded — is bounded and // acceptable: such a charge is created BY the very replay whose result this // discriminates, and the B1 in-flight guard + b1_attempts cap + refund re-poll // continue to bound any residual duplicate; the wrongly-refunded-legit-retry // direction this tolerance removes is unbounded by anything. const replayRescueUpperBoundSkew = 5 * time.Second // replayMatchesRowAmount reports whether the replayed payment charged the same // amount the pending row records — the amount the sweep's replay body repeats // and the amount any same-key retry MUST reuse (the retry path rejects a // different amount). A replayed payment carrying a DIFFERENT amount cannot be // the charge this row is waiting on and must never be rescued onto it. A // payment with no amount (zero — test fixtures; real Square payments always // carry one) is not refused here: the lag window below is the primary guard. func replayMatchesRowAmount(r staleRow, pr *square.PaymentResult) bool { return pr == nil || pr.Amount == 0 || pr.Amount == r.AmountPence } // replayWithinLegitimateWindow reports whether a replayed COMPLETED payment is // the REAL charge this pending row is waiting on — the ORIGINAL charge under a // retained key (created ~at row creation) or a later SAME-KEY RETRY charge // (created between the row's creation and the sweep's replay, F2). The amount // must match the row (a retry can never change it) and the payment must have // been created BEFORE the sweep's own replay (replayAt): a payment created at // or after replayAt can only be the sweep's own expired-key replay-induced // charge (the sweep just created it), so it is NOT legitimate. The source is // matched by construction: the replay body is rebuilt from the row's stored // square_request_snapshot with the LIVE square_source_id override, so a // payment returned by the replay necessarily charged the row's source (Square's // PaymentResult does not echo the source id back, so it cannot be compared // directly). func replayWithinLegitimateWindow(r staleRow, replayAt time.Time, pr *square.PaymentResult) bool { if !replayMatchesRowAmount(r, pr) { return false } created, ok := parseReplayedCreatedAt(pr) if !ok || r.CreatedAt.IsZero() { return false } // Lower bound: replayRescueLowerBoundSkew BEFORE the row — a DB clock ahead // of Square's can make a retained-key dedup return the ORIGINAL charge with // created_at slightly before the pending row. Such a payment cannot be a // provably-new expired-key replay (those land AT the sweep's replay, well // after the row), so within the skew tolerance it is the legitimate // original and must be rescued — not left pending to strand the customer's // authorized charge (Loop B MEDIUM). if created.Before(r.CreatedAt.Add(-replayRescueLowerBoundSkew)) { return false } // Upper bound: the sweep's OWN replay instant. A payment created strictly // before the replay is the original or a legit retry; created at/after the // replay is the sweep's own creation (expired-key duplicate). Comparing // against replayAt instead of a fixed row-age window makes the decision // independent of Square's unverified ~24h key-retention window. The small // replayRescueUpperBoundSkew tolerance keeps a legit same-key retry that // raced the sweep from being misread as the sweep's own replay-created // charge when Square's clock runs a few seconds ahead of the app's (MINOR). return created.Before(replayAt.Add(replayRescueUpperBoundSkew)) } // isSavedCardSource reports whether a Square source id is a card-on-file // (saved-card) reference. Only a ccof: source stays valid for recharging long // after the charge attempt that stored it: a cnon: nonce is single-use, so an // expired-key replay of a cnon: source is always rejected (proof the charge // never happened), while a replay of a still-valid ccof: source can land a NEW // charge under the expired key. func isSavedCardSource(source string) bool { return strings.HasPrefix(source, "ccof:") } // parseReplayedCreatedAt parses a PaymentResult's ISO 8601 CreatedAt into the // instant the replayed payment was created at Square. Real Square always // returns created_at on a payment, so an empty or unparseable value is a // client/response anomaly. func parseReplayedCreatedAt(pr *square.PaymentResult) (time.Time, bool) { if pr == nil || pr.CreatedAt == "" { return time.Time{}, false } created, err := time.Parse(time.RFC3339, pr.CreatedAt) if err != nil { return time.Time{}, false } return created, true } // replayRevealsNewCharge reports whether a COMPLETED keyed replay returned a // NEW charge rather than the ORIGINAL payment under a retained idempotency key // (finding A1). A retained-key dedup returns the original payment, created at // the same instant the pending row was created; a NEW charge made by an // expired-key replay (Square's ~24h key retention is UNVERIFIED — // square_http_client.go:626) against the still-valid ccof: source is created // AT THE SWEEP'S OWN REPLAY MOMENT. A same-key RETRY (handlers.go:1579-1591) // is a legitimate exception: the retry's charge is created between the row's // creation and the sweep's replay, so a replayed payment created BEFORE the // sweep's replay is the REAL charge and must be rescued (F2). Refusal is // money-safe: a replayed payment that cannot be proven to be the original (or // a retry created before the sweep's replay) is never rescued (the row stays // pending, a CRITICAL log is raised and an admin notification inserted), so a // hidden second charge can never masquerade as the original one. The lower // bound of the legitimate window is replayRescueLowerBoundSkew BEFORE the row: // a DB clock ahead of Square's can make the retained-key original look slightly // older, and such a payment is the legitimate original, not a new charge. // // The discriminator is the SWEEP'S OWN REPLAY TIMESTAMP (replayAt), captured // immediately before SquareClient.ReplayPaymentByKey. This is deliberately NOT // a fixed row-age window: previous iterations set a constant (22h then 24h, // matching stalePendingKeyedAge) and each review flipped it because the true // cutoff is "when did THIS sweep replay the key" — a payment created at/after // replayAt can only be the sweep's own expired-key creation, and a payment // created before it is the original or a legit retry, independent of Square's // actual (unverified) retention window. // // The check runs ONLY against real Square timestamps: it is gated off in an // explicit dev/mock env because the dev mock returns payments whose CreatedAt // is the mock's "now" at seed/replay time, uncorrelated with the aged // created_at the test rows carry (the keyed-reconcile tests age rows 23h while // seeding the Square payment at test time, and a retained-key dedup returns // that seeded payment). The gate mirrors the snapshot-decryption gate, which // also runs only in a non-dev/mock env. // // MONEY-DECISION PARITY (FIX 1, MINOR — pinned by // TestReplayRevealsNewCharge_DevMockGate_MoneyDecisionParity): the gate's // dev/mock outcome — "not a new charge", rescue the row — is IDENTICAL to what // the ungated prod classifier decides for every payment the dev mock can // actually return from ReplayPaymentByKey. The mock has NO key-retention / // key-expiry concept: a replay of a key already in its ledger dedups to the // ORIGINAL payment (created ~at row-creation time — inside the legitimate // window, so prod would also rescue it), and the only other COMPLETED outcome // is a fresh charge the mock creates AT the replay instant for an unknown key — // which lands within replayRescueUpperBoundSkew (5s) of replayAt and is // therefore still legitimate to prod (the same-key-retry tolerance). The gate // short-circuits ONLY the prod-branch scenarios the mock cannot produce — an // UNPARSEABLE created_at (prod: leave the row PENDING with a CRITICAL // notification) and a payment created beyond the upper-bound skew (prod: B1 // auto-refund the provable duplicate) — so the dev sweep path deliberately // does NOT exercise those prod branches. If the mock ever gains a key-expiry // simulation that returns a payment created beyond the skew, the pin test // fails and this gate MUST be revisited (prod and dev would diverge on the // money decision for a provably-new expired-key charge). func replayRevealsNewCharge(r staleRow, replayAt time.Time, pr *square.PaymentResult) (newCharge bool, created time.Time, createdOK bool) { if IsExplicitDevOrMockEnv() { return false, time.Time{}, false } created, createdOK = parseReplayedCreatedAt(pr) if !createdOK || r.CreatedAt.IsZero() { // Cannot prove the replayed payment is the original charge — refuse to // rescue rather than hide a possible second charge. return true, created, createdOK } return !replayWithinLegitimateWindow(r, replayAt, pr), created, true } // reconcileStalePaymentByKey asks Square for the authoritative status of the // charge made under a stale pending row's idempotency key and returns the // tri-state result. The replay sends an IDENTICAL body to the original charge: // the stored square_request_snapshot (the FULL request — source_id, key, // amount and every field the original carried), with the source_id overridden // by the row's CURRENT square_source_id column value (authoritative — a // same-key reuse refreshes the column while the snapshot JSON is stale). // Replaying a partial body would // return IDEMPOTENCY_KEY_REUSED for a RETAINED key and strand the row pending // forever. Square's idempotency guarantee returns the ORIGINAL payment for a // retained key (never a second charge); a COMPLETED payment rescues the row to // 'completed' with its real payment id. A definitive 4xx rejection // (ErrReplayKeyNotRetained — Square attempted a real charge with the // expired/used source and refused it, so the charge never happened) or a // FAILED/CANCELED payment proves the charge never completed and fails the row. // IDEMPOTENCY_KEY_REUSED is NEVER proof of no charge: with an identical body it // can only mean the stored source differs from the original (a data bug), so // the row is left pending with a CRITICAL log. Any OTHER error (transport / // 5xx / ambiguous) leaves the row pending — the charge may still have completed // at Square. The second return value is the Square payment id of the completed // payment ("" otherwise), written back on a rescue. // // ENV CONTRACT (finding 4): the sweep and the charge process MUST run with the // SAME SQUARE_ENVIRONMENT and SQUARE_LOCATION_ID. The stored snapshot embeds // the location_id used at charge time; if the env/location changes between a // charge and its replay, the replayed wire body differs and a RETAINED key // returns IDEMPOTENCY_KEY_REUSED — stranding every retained-key row pending // (safe) until the 24h blind-fail. This is a single-location deployment; the // contract is enforced by ops (same env for the sweeper and the API). The // environment and location are read through square.SquareEnvironment() and // square.SquareLocationID() — the SAME code path the charge-time HTTP client // uses (newHTTPClient resolves its base URL and location from those helpers) — // so the sweep can never drift to a second, independent env read; the // reconcile below logs the resolved values at each replay as a tripwire. func reconcileStalePaymentByKey(ctx context.Context, table string, r staleRow) (staleReconcileResult, string) { snapshot := r.SquareRequestSnapshot // fallbackBody is true when the row has NO stored square_request_snapshot // and the minimal body (key + source + amount) is rebuilt. The rebuilt body // omits reference_id / customer_id / note / buyer_email_address that the // original charge carried, so for a RETAINED key real Square compares the // WHOLE body, sees the difference and returns IDEMPOTENCY_KEY_REUSED // (stranding the row pending) while the dev mock — which compares only the // source — returns the original payment (rescuing it). Both are money-safe; // the divergence is dev-vs-prod OBSERVABILITY only, and pre-launch there // are no prod legacy rows (every charge now stores its snapshot), so the // fallback semantics are deliberately left unchanged. fallbackBody := false if len(bytes.TrimSpace(snapshot)) == 0 { fallbackBody = true // Legacy row without a stored request snapshot — rebuild the minimal // identical body (key + source + amount) exactly as the pre-snapshot // replay did. Such rows can still be reconciled as long as the stored // source/amount match the original charge. fallback, mErr := json.Marshal(square.CreatePaymentReq{ Amount: r.AmountPence, Currency: "GBP", SourceID: r.SquareSourceID, IdempotencyKey: r.IdempotencyKey, }) if mErr != nil { log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body for row %s (%v) — leaving pending", table, r.ID, mErr) return staleReconcileLeavePending, "" } snapshot = fallback } // PROD-mode snapshot decryption: since the AES-GCM snapshot work, the stored // square_request_snapshot is ENCRYPTED in a non-mock deployment // (SQUARE_ENVIRONMENT production/sandbox — the same gate the 2FA // enforcement uses, IsExplicitDevOrMockEnv); the dev mock stores plaintext. // Never replay a corrupt snapshot: a decryption failure is a serious error // that leaves the row pending with a CRITICAL log for manual reconciliation // instead of replaying garbage (which could return a misleading answer). if !fallbackBody && !IsExplicitDevOrMockEnv() { dec, err := decryptSnapshot(snapshot) if err != nil { return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: failed to decrypt the stored request snapshot for row %s (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", table, r.ID, err) } snapshot = dec } // The stored snapshot embeds the source_id of the ORIGINAL charge, but a // pending row REUSED by a same-key retry has its square_source_id column // refreshed to the retry's source while the snapshot JSON stays stale (the // write side now refreshes the snapshot too). Replaying the stale embedded // source under a key Square already retains for the new one would return // IDEMPOTENCY_KEY_REUSED and strand the row pending — so rebuild the replay // body with the LIVE column value, which is authoritative (set at charge // time, refreshed on every reuse). The minimal snapshot-less fallback body // above is already built from the live source and needs no override. An // unparseable snapshot must never look like proof of no charge — leave the // row pending for manual reconciliation. if !fallbackBody && r.SquareSourceID != "" { var req square.CreatePaymentReq if err := json.Unmarshal(snapshot, &req); err != nil { log.Printf("Stale pending %s reconcile by key: failed to parse stored request snapshot for row %s to override the source_id (%v) — leaving pending", table, r.ID, err) return staleReconcileLeavePending, "" } req.SourceID = r.SquareSourceID overridden, mErr := json.Marshal(req) if mErr != nil { log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body with the current square_source_id for row %s (%v) — leaving pending", table, r.ID, mErr) return staleReconcileLeavePending, "" } snapshot = overridden } // The replay repeats the stored request snapshot — with the live source_id // override above — so Square's idempotency dedup returns the original // payment for a retained key. The // sweep resolves its Square environment and location through the same // helpers the charge-time HTTP client uses (square.SquareEnvironment / // square.SquareLocationID), so the replay and the charge can never read // two different env sources (see the ENV CONTRACT above). The values are // logged on each replay as a tripwire for env/location drift between the // sweeper and the API. log.Printf("[SWEEP] replay-by-key reconcile for %s row %s: SQUARE_ENVIRONMENT=%q SQUARE_LOCATION_ID=%q (must match the charge-time env/location for identical-body idempotency)", table, r.ID, square.SquareEnvironment(), square.SquareLocationID()) // The discriminator for "original/retry vs expired-key duplicate" is the // sweep's OWN replay instant: any payment Square returns with created_at // at/after this moment was created by THIS replay call (the key expired and // Square charged the still-valid source), so it must never be rescued. A // payment created before this instant is the original or a legit same-key // retry. Capturing replayAt here — not a fixed row-age window — keeps the // decision correct regardless of Square's unverified ~24h retention. replayAt := clock.Now() pr, err := SquareClient.ReplayPaymentByKey(ctx, snapshot) if err != nil { if errors.Is(err, square.ErrReplayKeyNotRetained) { // A1: a rejection of a replayed charge against a SAVED-CARD (ccof:) // source is NOT proof the original charge never happened — the same // key-retention race that lets the replay land a NEW charge can // leave that charge even when the probe returns a rejection. Blindly // failing (and clawing back a till sale's funded gift card) on a // ccof source could reverse money a replay-created charge already // took. Leave the row pending and alert ops. A cnon-source // (spent-nonce) rejection REMAINS definitive proof of no charge (a // single-use nonce cannot be recharged), keeping the proven-failed // path and its clawback. if isSavedCardSource(r.SquareSourceID) { return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: Square rejected the identical-body replay (no payment under the stored key) but the row's source is a still-valid saved card (ccof:) — the replay may have landed a NEW charge under an expired idempotency key — leaving row %s PENDING without failing/clawing back — MANUAL RECONCILIATION REQUIRED: verify at Square whether a charge exists before re-issuing", table, r.ID) } log.Printf("Stale pending %s reconcile by key: Square has no payment under the stored idempotency key (identical-body replay rejected) — marking failed; the charge provably never happened", table) return staleReconcileDefinitivelyFailed, "" } if square.ErrorCode(err) == "IDEMPOTENCY_KEY_REUSED" { if fallbackBody { // Snapshot-less fallback row: the minimal rebuilt body cannot be // identical to the original charge, so real Square rejects the // retained-key replay while the dev mock (source-aware only) // rescues it. The stranded row would otherwise be invisible // until the 24h blind-fail — surface it now (finding 2). return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED via the minimal snapshot-less fallback body (key=%s) — real Square compares the WHOLE request (reference_id/customer_id/note/buyer_email_address absent from the rebuilt body) and rejects the identical-key replay, stranding the row pending; the dev mock would rescue it — dev-vs-prod observability divergence, NOT proof the charge never happened — MANUAL RECONCILIATION REQUIRED", table, r.IdempotencyKey) } return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED — the stored square_source_id differs from the original charge's source (data bug); this is NOT proof the charge never happened — leaving pending — MANUAL RECONCILIATION REQUIRED", table) } log.Printf("Stale pending %s reconcile by idempotency key hit an ambiguous error (%v) — leaving pending for a later sweep run", table, err) return staleReconcileLeavePending, "" } // Square's terminal payment states are COMPLETED, CANCELED, FAILED; // APPROVED (authorization-only, delayed capture) and PENDING are // NON-terminal — both can still transition to COMPLETED, so clawing back // the funded gift card on either would risk reversing a charge that later // lands. This app always creates payments with autocomplete (default true), // so it never produces APPROVED/PENDING rows today, but the classification // must match Square's documented state machine. switch pr.Status { case "COMPLETED": // A1: a replayed COMPLETED payment created long AFTER the pending row // is a NEW charge Square made against the still-valid source with an // expired idempotency key (Square's ~24h key retention is UNVERIFIED — // square_http_client.go:626), NOT the original charge a retained key // returns. Rescuing the row with the new payment id would hide the // second charge behind the original. Leave the row pending and alert // ops so both charges can be reconciled at Square and the duplicate // refunded. if newCharge, created, createdOK := replayRevealsNewCharge(r, replayAt, pr); newCharge { // B1 caveat: auto-refund ONLY a PROVABLY-created-later duplicate. // An UNPARSEABLE replayed CreatedAt does not prove the payment is a // duplicate — it could be the ORIGINAL charge a retained key // returned, whose created_at was lost/corrupt. Auto-refunding that // would wrongly reverse a legitimately authorized charge, so the // pre-B1 outcome is restored for this case: leave the row PENDING // with a CRITICAL notification for a human to reconcile at Square // (the refund is withheld until a human confirms the second charge // exists — never auto-issued against a possibly-original payment). if !createdOK { return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s has an UNPARSEABLE created_at — cannot prove it is a NEW expired-key replay rather than the ORIGINAL charge under a retained key — leaving row %s PENDING — MANUAL RECONCILIATION REQUIRED: verify at Square whether this is a second charge before refunding", table, pr.ID, r.ID) } // B1 (MEDIUM 2): a replayed COMPLETED payment created BEFORE the // pending row is NOT provably a new expired-key replay — those land // ~22h AFTER the row. Only genuinely-before payments reach this // line: a payment within replayRescueLowerBoundSkew of the row is // already treated as the ORIGINAL retained-key charge by // replayWithinLegitimateWindow and rescued. This one is far enough // before the row that no clock skew can explain it — the row is // left PENDING with a CRITICAL notification for manual // reconciliation, never auto-refunded. if created.Before(r.CreatedAt) { return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s was created %s before the pending row %s (beyond the %s clock-skew tolerance) — cannot prove it is a NEW expired-key replay nor the ORIGINAL charge — leaving row %s PENDING — MANUAL RECONCILIATION REQUIRED: verify at Square whether this is a second charge before refunding", table, pr.ID, created.Sub(r.CreatedAt).Round(time.Minute), r.ID, replayRescueLowerBoundSkew, r.ID) } lag := created.Sub(r.CreatedAt).Round(time.Minute).String() // B1: the replayed COMPLETED payment is a REAL charge the customer // never authorized (an expired-key replay landed it on the still // valid saved-card source, provably created after the pending row). // Auto-refund C2 at Square instead of leaving the row pending for a // human: the amount is r.AmountPence // (the replay repeats the row's amount), the key is fresh, and the // reason marks the sweep. When the refund lands, the row is marked // failed (the original charge was never found — the caller's // definitively-failed branch does that; a till sale's gift-card // funding is clawed back there too, since the duplicate has been // refunded). A refund Square leaves PENDING is NON-terminal: the row // stays pending (no fail, no clawback) and the B1 re-poll pass // resolves it when Square settles. Only a refund FAILURE keeps the // CRITICAL manual-reconciliation path. if refundErr := refundSweepDuplicateCharge(ctx, table, r, pr); refundErr == nil { log.Printf("stale pending %s row %s: the replayed COMPLETED payment %s was a NEW charge under an expired idempotency key (lag %s) — auto-refunded the duplicate at Square and marking the row failed (the original charge was never found)", table, r.ID, pr.ID, lag) return staleReconcileDefinitivelyFailed, "" } else if errors.Is(refundErr, errSweepRefundPending) { log.Printf("stale pending %s row %s: the auto-refund of the replay-induced duplicate charge %s is PENDING at Square (non-terminal) — leaving the row pending; the B1 refund re-poll resolves it when Square settles", table, r.ID, pr.ID) return staleReconcileLeavePending, "" } else if errors.Is(refundErr, errSweepRefundRejected) { // B1 (CRITICAL-HIGH): the auto-refund was DEFINITIVELY REJECTED // at Square — the duplicate charge stands and can never be // auto-refunded. The parent row is failed WITHOUT a gift-card // clawback (the money is real at Square), the CRITICAL // notification is raised, and b1_attempts is set to the cap so // the expired key is never replayed (each replay would mint // ANOTHER charge). The outer loop's // staleReconcileDefinitivelyFailedNoClawback branch does all of // this. log.Printf("stale pending %s row %s: the auto-refund of the replay-induced duplicate charge %s was DEFINITIVELY REJECTED at Square (%v) — marking the row failed with a CRITICAL notification; the expired key will never be replayed", table, r.ID, pr.ID, refundErr) return staleReconcileDefinitivelyFailedNoClawback, "" } else { // Loop-B finding (MED): the auto-refund ERRORED (transport / // any non-pending, non-rejected error) — the money state at // Square is unknown, NO refunds row was written, and the row's // b1_attempts was already incremented. The OLD code left the // row pending, so the next run re-replayed the SAME expired key // and minted ANOTHER charge — the b1_attempts cap of 3 allowed // up to 3 stacked unauthorized charges. Do NOT re-replay a key // whose refund attempt errored: fail the parent row + CRITICAL // immediately (the NoClawback branch — the duplicate charge may // stand at Square, so a till sale's funding is NOT clawed back; // the b1_attempts cap is pinned so the key is never replayed). // A genuinely-ambiguous refund (accepted by Square, left PENDING) // is already handled by the B1 re-poll pass. log.Printf("stale pending %s row %s: the auto-refund of the replay-induced duplicate charge %s ERRORED (%v) — marking the row failed with a CRITICAL notification; the expired key will never be replayed", table, r.ID, pr.ID, refundErr) return staleReconcileDefinitivelyFailedNoClawback, "" } } return staleReconcileCompleted, pr.ID case "CANCELED", "FAILED": log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed, "" case "APPROVED", "PENDING": log.Printf("Stale pending %s is %q at Square (replay by key, non-terminal) — leaving pending for a later sweep run", table, pr.Status) return staleReconcileLeavePending, "" default: log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed, "" } } // errSweepRefundPending is the sentinel refundSweepDuplicateCharge returns when // Square accepted the auto-refund of a replay-induced duplicate charge but left // it PENDING — a NON-terminal state: the refund may still complete (the // duplicate is reversed) or fail. The caller must NOT treat it as success: the // parent row is left PENDING (never marked failed, a till sale's funded gift // card is never clawed back) so a later sweep run can re-poll the refund. The // refund's idempotency key ("sweepdup-"+paymentID) is deterministic, so the // re-poll finds the SAME refund at Square. var errSweepRefundPending = errors.New("sweep duplicate-charge refund pending at Square") // errSweepRefundRejected is the sentinel refundSweepDuplicateCharge returns // when Square DEFINITIVELY rejected the auto-refund of a replay-induced // duplicate charge (refund status FAILED/REJECTED). The duplicate charge // stands at Square and can never be auto-refunded, so the caller must mark the // parent row failed, raise the CRITICAL notification and set b1_attempts to // the cap — replaying the expired key would mint ANOTHER charge (CRITICAL-HIGH // B1). Unlike errSweepRefundPending (non-terminal, row stays pending), this is // terminal and MUST NOT be left for the next sweep run. var errSweepRefundRejected = errors.New("sweep duplicate-charge refund rejected at Square") // sweepDuplicateRefundReason is the audit-trail reason carried by every // refunds row for a sweep auto-refund of a replay-induced duplicate charge // (B1). The B1 re-poll pass (sweepPendingB1Refunds, refunds.go) and the // in-flight guard (hasInFlightSweepDuplicateRefund) match on it, so it must // stay in lockstep with refunds.go. For a till_sale the reason carries the // parent till_sale id (see sweepDuplicateRefundReasonFor) — refunds has no // till_sale column and the refund must link back to the sale the re-poll pass // claws back when Square settles. const sweepDuplicateRefundReason = "duplicate charge — sweep replay" // sweepDuplicateRefundReasonFor returns the refunds-row reason for a sweep // auto-refund. tillSaleID is "" for payments-table rows (whose parent is the // refund's own payment_id); a till_sale's parent is encoded in the reason // because refunds.payment_id is FK'd to payments and the sale has no payments // row of its own. func sweepDuplicateRefundReasonFor(tillSaleID string) string { if tillSaleID == "" { return sweepDuplicateRefundReason } return sweepDuplicateRefundReason + " (till_sale " + tillSaleID + ")" } // refundSweepDuplicateCharge auto-refunds a replayed COMPLETED payment that the // sweep proved to be a NEW charge under an expired idempotency key (B1) — money // the customer never authorized. The refund reuses the existing Square refund // path (SquareClient.RefundPayment) with: // // - amount pr.Amount — the replayed payment's ACTUAL charged amount. The row's // stored amount (r.AmountPence) can differ when the original charge applied // a deposit-with-discount (chargeAmount = requested - discount): the row // records the discounted charge, and the replay repeats that discounted // body, so the duplicate charged exactly pr.Amount; // - a deterministic idempotency key derived from the replayed payment's id, // so a re-run refunding the SAME duplicate dedups at Square instead of // issuing a second refund, while a different duplicate on a later replay // gets a fresh key and is refunded too; // - reason "duplicate charge — sweep replay" for the audit trail. // // A refunds row is recorded for BOTH payments-table and till_sale rows (a // till_sale's refund attaches to a synthetic payments row created for the // duplicate charge, since refunds.payment_id is FK'd to payments; the insert // is idempotent on the deterministic refund key, whose UNIQUE constraint // doubles as the dedup guard), and an admin notification is inserted so an // operator sees the auto-refund. // // Return semantics: // // - nil — the refund COMPLETED at Square; the caller marks the row // definitively failed (the ORIGINAL charge was never found) and, for a till // sale, claws back the funded gift card — the duplicate has been reversed; // - errSweepRefundPending — the refund is PENDING (non-terminal); the caller // leaves the row pending; the B1 re-poll pass (sweepPendingB1Refunds, // refunds.go) resolves the row when Square settles; // - any other error — the money state at Square is untouched; the caller // keeps the CRITICAL manual-reconciliation path. func refundSweepDuplicateCharge(ctx context.Context, table string, r staleRow, pr *square.PaymentResult) error { if pr == nil || pr.ID == "" { return errors.New("replayed payment has no Square payment id to refund") } if pr.Amount <= 0 { return fmt.Errorf("refusing to auto-refund a non-positive amount %d pence for replayed payment %s (row %s)", pr.Amount, pr.ID, r.ID) } // B1: count THIS attempt against the row's b1_attempts cap so a refund // that keeps failing (REJECTED at call time writes no refunds row; a // transport error writes none either) can never be replayed forever. // Each replay mints another charge at Square. incrementB1Attempts(ctx, table, r.ID) // Deterministic per-duplicate key: Square dedups same-key refunds, so a // re-run replaying the same C2 never double-refunds. "sweepdup-" + Square // payment id stays well under Square's 45-char idempotency-key limit. refundKey := "sweepdup-" + pr.ID if len(refundKey) > maxIdempotencyKeyLength { refundKey = truncateIdempotencyKey("sweepdup", pr.ID) } res, refundErr := SquareClient.RefundPayment(ctx, square.RefundPaymentReq{ PaymentID: pr.ID, Amount: pr.Amount, IdempotencyKey: refundKey, Reason: sweepDuplicateRefundReason, }) if refundErr != nil { return fmt.Errorf("auto-refund of replay-induced duplicate charge %s failed: %w", pr.ID, refundErr) } status := "completed" pending := false switch res.Status { case "PENDING", "APPROVED": // Square's non-terminal refund states — the money has not moved yet but // the refund is in flight. NON-terminal: the caller must leave the row // pending and never claw back a funded gift card (reversing the funding // before Square settles the refund could leave a customer charged with // no gift card if the refund later fails). status = "pending" pending = true case "FAILED", "REJECTED": // Square definitively rejected the refund — the duplicate charge stands // and can never be auto-refunded. The caller must NOT leave the row // pending for another replay (a replay mints ANOTHER charge): it marks // the parent failed, raises the CRITICAL notification and sets the // b1_attempts cap so the expired key is never replayed. return fmt.Errorf("auto-refund of replay-induced duplicate charge %s was %s at Square: %w", pr.ID, res.Status, errSweepRefundRejected) } recordSweepDuplicateRefundRow(ctx, table, r, pr, res.ID, status, refundKey) insertCriticalPaymentNotification(ctx, r.BookingID, r.CreatedBy) if pending { log.Printf("Auto-refund of replay-induced duplicate charge %s (%d pence) for pending row %s is PENDING at Square (refund %s) — leaving the row pending for the refund re-poll", pr.ID, pr.Amount, r.ID, res.ID) return errSweepRefundPending } log.Printf("Auto-refunded replay-induced duplicate charge %s (%d pence) for pending row %s — refund %s", pr.ID, pr.Amount, r.ID, res.ID) return nil } // recordSweepDuplicateRefundRow writes the refunds row for a sweep auto-refund // of a replay-induced duplicate charge (B1), idempotently on the deterministic // refund idempotency key (UNIQUE — a re-run can never mint a second row). For // a payments-table row the refund attaches to the pending payment row itself. // A till_sale has no payments row (refunds.payment_id is NOT NULL + FK), so a // synthetic payments row is created FIRST — 'completed' (the duplicate charge // genuinely landed at Square) with the duplicate's square_payment_id, so the // B1 re-poll pass can look the refund up by payment id — and the refund is // attached to it. The parent till_sale id is carried in the reason so the // re-poll pass claws it back when Square settles. func recordSweepDuplicateRefundRow(ctx context.Context, table string, r staleRow, pr *square.PaymentResult, refundID, status, refundKey string) { amountPounds := float64(pr.Amount) / 100.0 paymentID := r.ID var bookingID *string reason := sweepDuplicateRefundReason if table == "payments" { bookingID = r.BookingID } else { // till_sale: create the synthetic payments row for the duplicate charge. payKey := truncateIdempotencyKey("sweepdup-pay", pr.ID) err := db.Conn.QueryRow(ctx, ` INSERT INTO payments (payment_type, payment_method, status, amount, square_payment_id, idempotency_key, created_by, created_at) VALUES ('full', 'in_person_card', 'completed', $1, $2, $3, $4, NOW()) ON CONFLICT (idempotency_key) DO NOTHING RETURNING id `, amountPounds, pr.ID, payKey, r.CreatedBy).Scan(&paymentID) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but creating the payments row for till sale %s failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, r.ID, err) return } // Re-run: the synthetic row already exists — reuse it. if rErr := db.Conn.QueryRow(ctx, `SELECT id FROM payments WHERE idempotency_key = $1`, payKey).Scan(&paymentID); rErr != nil { log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but re-reading the payments row for till sale %s failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, r.ID, rErr) return } } reason = sweepDuplicateRefundReasonFor(r.ID) } if _, insErr := db.Conn.Exec(ctx, ` INSERT INTO refunds (payment_id, booking_id, amount, square_refund_id, status, origin, reason, idempotency_key, created_by, created_at) VALUES ($1, $2, $3, $4, $5, 'manual', $6, $7, $8, NOW()) ON CONFLICT (idempotency_key) DO UPDATE SET status = EXCLUDED.status, square_refund_id = EXCLUDED.square_refund_id `, paymentID, bookingID, amountPounds, refundID, status, reason, refundKey, r.CreatedBy); insErr != nil { log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but recording the refunds row failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, insErr) } } // hasInFlightSweepDuplicateRefund reports whether a stale pending row carries a // sweep auto-refund of a replay-induced duplicate charge (B1) that Square left // PENDING — or that a webhook promoted to 'completed' while the parent is still // pending. While the refund is in flight the row must NOT be replayed (a replay // of the expired-key row could land ANOTHER charge), blind-failed or clawed // back (the refund is non-terminal — money may still reverse). Loop-B finding: // matching only 'pending' let a webhook-promoted 'completed' refund fall off // the guard, so the next sweep re-replayed the expired key and minted a second // charge before the re-poll pass could resolve the parent. The parent being // still pending is implied: the sweep only processes pending rows. The B1 // re-poll pass (sweepPendingB1Refunds, refunds.go) resolves the parent row when // Square settles. For a till_sale the refund row's reason carries the sale id // (see sweepDuplicateRefundReasonFor); a payments-table refund's payment_id IS // the parent row. func hasInFlightSweepDuplicateRefund(ctx context.Context, table, id string) bool { var exists bool var err error if table == "till_sales" { err = db.Conn.QueryRow(ctx, ` SELECT EXISTS ( SELECT 1 FROM refunds WHERE status IN ('pending', 'completed') AND square_refund_id IS NOT NULL AND reason = $1 ) `, sweepDuplicateRefundReasonFor(id)).Scan(&exists) } else { err = db.Conn.QueryRow(ctx, ` SELECT EXISTS ( SELECT 1 FROM refunds WHERE payment_id = $1 AND status IN ('pending', 'completed') AND square_refund_id IS NOT NULL AND reason = $2 ) `, id, sweepDuplicateRefundReason).Scan(&exists) } if err != nil { log.Printf("Failed to check for an in-flight sweep duplicate refund on %s row %s: %v", table, id, err) return false } return exists } // hasFailedSweepDuplicateRefund reports whether a stale pending row carries a // sweep auto-refund of a replay-induced duplicate charge (B1) that Square // definitively REJECTED (refunds row status 'failed'). The FAILED-webhook // demotion (refund.updated → status failed) or the B1 re-poll's terminal-failed // resolution marks a refund that was accepted PENDING and later failed — which // clears the in-flight guard (hasInFlightSweepDuplicateRefund only matches // 'pending'). Replaying such a row's expired key would mint ANOTHER charge, so // the sweep must never do it (CRITICAL-HIGH B1). A till_sale's reason carries // the sale id (see sweepDuplicateRefundReasonFor), mirroring the in-flight // guard's scoping. func hasFailedSweepDuplicateRefund(ctx context.Context, table, id string) bool { var exists bool var err error if table == "till_sales" { err = db.Conn.QueryRow(ctx, ` SELECT EXISTS ( SELECT 1 FROM refunds WHERE status = 'failed' AND square_refund_id IS NOT NULL AND reason = $1 ) `, sweepDuplicateRefundReasonFor(id)).Scan(&exists) } else { err = db.Conn.QueryRow(ctx, ` SELECT EXISTS ( SELECT 1 FROM refunds WHERE payment_id = $1 AND status = 'failed' AND square_refund_id IS NOT NULL AND reason = $2 ) `, id, sweepDuplicateRefundReason).Scan(&exists) } if err != nil { log.Printf("Failed to check for a failed sweep duplicate refund on %s row %s: %v", table, id, err) return false } return exists } // incrementB1Attempts records one more B1 auto-refund attempt against a stale // pending row's b1_attempts cap. Runs BEFORE the Square refund call so every // attempt (success, pending, rejected, transport error) counts — a refund that // keeps failing without writing a refunds row can never exceed the cap and // force an unbounded replay loop. Best-effort: a failed UPDATE must not block // the refund itself. func incrementB1Attempts(ctx context.Context, table, id string) { if _, err := db.Conn.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET b1_attempts = b1_attempts + 1, updated_at = NOW() WHERE id = $1 `, id); err != nil { log.Printf("Failed to increment b1_attempts on %s row %s: %v", table, id, err) } } // setB1AttemptsToCap pins a stale pending row's b1_attempts to the cap after a // B1 auto-refund was definitively REJECTED at Square — the belt-and-suspenders // guarantee that the expired key is never replayed even if the row somehow // stays pending. Best-effort like incrementB1Attempts. func setB1AttemptsToCap(ctx context.Context, table, id string) { if _, err := db.Conn.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET b1_attempts = $2, updated_at = NOW() WHERE id = $1 AND b1_attempts < $2 `, id, b1DuplicateRefundAttemptCap); err != nil { log.Printf("Failed to set b1_attempts cap on %s row %s: %v", table, id, err) } } // leaveGiftCardPurchasePending keeps a gift-card-purchase payment row (payments // table, booking_id NULL) pending after Square confirms the charge COMPLETED, // instead of rescuing it to 'completed'. Completing the row would permanently // block the same-key retry (BuyGiftCard) that reuses the pending record to // deliver the card — the customer would stay charged with no gift card (C6). // The row is left pending with a CRITICAL log + admin notification so the retry // can still deliver the card and an admin is alerted to reconcile manually. func leaveGiftCardPurchasePending(ctx context.Context, r staleRow) { log.Printf("CRITICAL: gift card purchase payment %s is COMPLETED at Square but the gift card was never issued (issue transaction failed, retry abandoned) — leaving the payment PENDING so a same-key retry can still deliver the card — MANUAL RECONCILIATION REQUIRED", r.ID) insertCriticalPaymentNotification(ctx, nil, r.CreatedBy) } // insertCriticalPaymentNotification surfaces an unresolved money event in the // admin notification centre (reason 'critical_payment_log' — the DB-backed // stand-in for the un-watched CRITICAL payment logs). bookingID is set when the // issue ties to a booking (untracked terminal charges); userID is set when it // ties to a user (gift-card purchases). The NOT EXISTS guard keeps ONE // notification per issue instead of one per sweep run, and requires the prior // notification to be unacknowledged (acknowledged_at IS NULL) so that after an // admin acknowledges it, a NEW event for the same booking/user re-notifies. // // This is the SWEEP-specific variant of a same-named helper in the webhooks // package (handlers/webhooks/square.go, insertCriticalPaymentNotification) with // an INTENTIONAL scoping difference: this one dedups on (reason, booking_id, // user_id) — booking and/or user — because sweep-originated events may be // user-attributed without a booking (gift-card purchases) or booking-attributed // without a user (untracked terminal charges). The webhook variant takes // (ctx, bookingID, disputeID string), never writes user_id, and dedups on // (reason, booking_id) or a deterministic per-dispute id. Both share the // admin_notifications table and the 'critical_payment_log' reason but serve // different call paths (sweep vs webhook) — do not merge them, and if the // NOT EXISTS guard shape ever changes, update BOTH copies. func insertCriticalPaymentNotification(ctx context.Context, bookingID, userID *string) { // Apply the SAME global flood cap as every other 'critical_payment_log' // insert site (webhooks, account erasure, jwt reuse, time-blockers, cleanup): // a fast-path suppression pre-check, then the cap folded INTO the INSERT. // // The count-then-insert critical section is serialized on a transaction- // scoped advisory lock so it is TRULY atomic (F6): under READ COMMITTED, N // concurrent inserts can each evaluate the COUNT subquery against a snapshot // taken before ANY of them commits, so all N pass the `< $cap` check and the // cap is overshot by N. pg_advisory_xact_lock makes each waiter's COUNT run // only AFTER the previous insert committed, so the unacknowledged queue // never exceeds the cap — the overshoot is bounded by exactly zero instead // of by the concurrency. The transaction is one short INSERT, so the // blocking xact lock (no 3s bound — see acquireAdvisoryXactLockBlocking) // is fine here. // // The per-issue NOT EXISTS dedup is preserved, so each issue still gets // exactly one notification. if adminnotify.CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") { slog.Warn("suppressed critical-payment admin notification — unacknowledged 'critical_payment_log' queue at the cap", "booking", bookingID, "user", userID) return } tx, err := db.Conn.Begin(ctx) if err != nil { log.Printf("Failed to begin critical-payment admin notification insert: %v", err) return } defer func() { if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { log.Printf("Failed to rollback critical-payment admin notification insert: %v", err) } }() if err := acquireAdvisoryXactLockBlocking(ctx, tx, "crussell:critical-payment-log-cap"); err != nil { log.Printf("Failed to acquire critical-payment admin notification cap lock: %v", err) return } tag, err := tx.Exec(ctx, ` INSERT INTO admin_notifications (reason, booking_id, user_id, created_at) SELECT 'critical_payment_log', $1, $2, NOW() WHERE (SELECT COUNT(*) FROM admin_notifications _an WHERE _an.reason = 'critical_payment_log' AND _an.acknowledged_at IS NULL) < $3 AND NOT EXISTS ( SELECT 1 FROM admin_notifications an WHERE an.reason = 'critical_payment_log' AND an.booking_id IS NOT DISTINCT FROM $1 AND an.user_id IS NOT DISTINCT FROM $2 AND an.acknowledged_at IS NULL ) `, bookingID, userID, adminnotify.MaxUnacknowledgedCriticalLogs) if err != nil { log.Printf("Failed to insert admin_notifications for critical payment issue (booking=%v user=%v): %v", bookingID, userID, err) return } if err := tx.Commit(ctx); err != nil { log.Printf("Failed to commit critical-payment admin notification insert: %v", err) return } if int(tag.RowsAffected()) > 0 { log.Printf("Inserted critical-payment admin notification (booking=%v user=%v)", bookingID, userID) } } // notifyStaleRowCritical inserts a critical-payment admin notification for a // stale row via the shared insertCriticalPaymentNotification helper — the // notification's deterministic dedup key is the user attribution (the NOT // EXISTS guard keeps ONE notification per user instead of one per sweep run). // Payments rows are attributed by the payer (created_by); till_sales rows by // the funded gift card's redeemed-to user when there is one. booking_id is // deliberately not used: the sweep runs after the charge process and a // booking-attributed notification would pin the booking row through the // admin_notifications FK (no cascade) for as long as the notification survives. func notifyStaleRowCritical(ctx context.Context, r staleRow) { var userID *string if r.CreatedBy != nil { userID = r.CreatedBy } else { userID = r.RedeemToUserID } insertCriticalPaymentNotification(ctx, nil, userID) } // leavePendingCritical is the shared terminal outcome for a stale pending row // that cannot be safely resolved by the sweep: it logs a CRITICAL line (the // caller's existing message), raises the deduped critical-payment admin // notification via notifyStaleRowCritical, and returns // staleReconcileLeavePending so the row stays pending for a human. Every // "do not touch, an operator must reconcile" branch — snapshot-decrypt // failure, the ccof replay rejection, IDEMPOTENCY_KEY_REUSED and the // replayed-new-charge case (A4) — now runs the identical log + notification // outcome through this one helper. func leavePendingCritical(ctx context.Context, r staleRow, format string, args ...any) (staleReconcileResult, string) { notifyStaleRowCritical(ctx, r) log.Printf("CRITICAL: "+format, args...) return staleReconcileLeavePending, "" } // staleReconcileResult is the tri-state outcome of reconciling one stale // pending row against Square. Only a definitively-resolved outcome touches the // row: an ambiguous answer (transport error / 5xx) leaves it pending so a // same-key retry can still reuse it if the charge actually completed. type staleReconcileResult int const ( // staleReconcileLeavePending — Square's answer was ambiguous; the row stays // pending for the next sweep run. staleReconcileLeavePending staleReconcileResult = iota // staleReconcileCompleted — Square confirms the charge completed; rescue // the row to 'completed'. staleReconcileCompleted // staleReconcileDefinitivelyFailed — Square proves the charge never // completed (payment not found / non-completed status); mark the row // 'failed' exactly as the legacy bulk sweep did. staleReconcileDefinitivelyFailed // staleReconcileDefinitivelyFailedNoClawback — Square definitively rejected // the B1 auto-refund of a replay-induced duplicate charge (the duplicate // stands at Square, so the money is REAL and a till sale's funded gift card // must NOT be clawed back). The outer loop marks the parent failed without a // clawback, raises the CRITICAL notification and sets the b1_attempts cap. staleReconcileDefinitivelyFailedNoClawback ) // reconcileStalePaymentAtSquare asks Square for the authoritative status of a // stale pending charge and returns the tri-state result. A COMPLETED payment // rescues the row to 'completed'; a NOT_FOUND error or any non-COMPLETED status // proves the charge never completed and fails the row as the legacy bulk sweep // did. Any OTHER error (transport / 5xx / ambiguous) is NOT treated as a // definitive failure — the charge may still have completed at Square, and // marking the row failed would close the double-charge window (blocking a // same-key retry with a 409) even though the money moved. Such rows stay // pending for a later run. func reconcileStalePaymentAtSquare(ctx context.Context, table, squarePaymentID string) staleReconcileResult { pr, err := SquareClient.GetPayment(ctx, squarePaymentID) if err != nil { if squarePaymentErrorIsNotFound(err) { log.Printf("Stale pending %s reconcile: Square payment %s not found (%v) — marking failed as the legacy sweep would", table, squarePaymentID, err) return staleReconcileDefinitivelyFailed } log.Printf("Stale pending %s reconcile for Square payment %s hit an ambiguous error (%v) — leaving pending for a later sweep run", table, squarePaymentID, err) return staleReconcileLeavePending } // Square's terminal payment states are COMPLETED, CANCELED, FAILED; // APPROVED (authorization-only, delayed capture) and PENDING are // NON-terminal — both can still transition to COMPLETED (via // CompletePayment or delay_action=COMPLETE), so clawing back the funded // gift card on either would risk reversing a charge that later lands. // This app always creates payments with autocomplete (default true), so // it never produces APPROVED/PENDING rows today, but the classification // must match Square's documented state machine (librarian-verified // 2026-05-20). switch pr.Status { case "COMPLETED": return staleReconcileCompleted case "CANCELED", "FAILED": log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed case "APPROVED", "PENDING": log.Printf("Stale pending %s is %q at Square (non-terminal) — leaving pending for a later sweep run", table, pr.Status) return staleReconcileLeavePending default: log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed } } // squarePaymentErrorIsNotFound reports whether a GetPayment error proves the // payment does not exist at Square. The structured not-found check is primary // (square.IsNotFound: NOT_FOUND code / HTTP 404 status / plain "HTTP 404" body); // the message fallback covers only the dev mock's plain "payment not found" // error, which carries no structured code. func squarePaymentErrorIsNotFound(err error) bool { if err == nil { return false } if square.IsNotFound(err) { return true } // Any other structured Square error code is authoritative — never // substring-match its message. if square.ErrorCode(err) != "" { return false } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") } // squareHasCode reports whether err carries a structured Square error code // equal to any of codes. Errors without a structured code (the dev mock's // plain errors) match nothing. func squareHasCode(err error, codes ...string) bool { if err == nil { return false } code := square.ErrorCode(err) for _, c := range codes { if code == c { return true } } return false } // sweepSquarelessManualRefundsAtAttemptCap reconciles Square-less MANUAL refund // rows that have already hit the attempt cap (maxManualRefundAttempts). The // Square-less pre-pass inside sweepManualPendingSquareRefunds (refunds.go) // filters refund_attempts < maxManualRefundAttempts, so a legacy manual refund // on a payment with no square_payment_id that reached the cap (attempts // incremented by pre-guard Square attempts) is never reconciled by it and would // stay 'pending' forever, permanently blocking the over-refund guard (F10). // Such rows can never be refunded via Square, so they are marked 'failed' and // surfaced in the admin notification centre for in-person arrangement — the // same terminal treatment the pre-pass gives rows under the cap. Returns the // number of rows marked failed. func sweepSquarelessManualRefundsAtAttemptCap(ctx context.Context) (int, error) { rows, err := db.Conn.Query(ctx, fmt.Sprintf(` UPDATE refunds r SET status = 'failed' FROM payments p WHERE p.id = r.payment_id AND r.status = 'pending' AND r.refund_attempts >= %d AND p.payment_method IN ('online_square', 'in_person_card') AND p.square_payment_id IS NULL AND r.origin = 'manual' RETURNING r.id `, maxManualRefundAttempts)) if err != nil { return 0, err } defer rows.Close() var failedIDs []string for rows.Next() { var id string if err := rows.Scan(&id); err == nil { failedIDs = append(failedIDs, id) } } if err := rows.Err(); err != nil { return 0, err } if len(failedIDs) > 0 { log.Printf("Marked %d Square-less manual refund(s) at the %d-attempt cap failed (in-person arrangement needed)", len(failedIDs), maxManualRefundAttempts) } insertRefundFailedNotifications(ctx, failedIDs) return len(failedIDs), nil } // staleTerminalCheckoutAge is how old a still-pending terminal checkout must // be before the sweep cancels it. Terminal checkouts normally complete within // minutes; an hour is far past any legitimate card-reader interaction while // still short enough that a completed charge can never be misread as stale. const staleTerminalCheckoutAge = 1 * time.Hour // SweepStaleTerminalCheckouts cancels terminal (card-machine) checkouts that // are still PENDING/IN_PROGRESS long after they were created, so the terminal // stops waiting on a customer who walked away. A checkout created by // CreateTerminalPayment / CreateTillSale that is never polled would otherwise // sit live at Square indefinitely; if it later completes it is an invisible, // untracked charge. // // Two tables track live checkout IDs and are both swept: // - terminal_checkouts: booking terminal checkouts created by // CreateTerminalPayment. PENDING/IN_PROGRESS rows older than the cutoff // are resolved at Square first: a checkout still waiting at Square is // cancelled and re-checked once — if it completed during the cancel window // the row is marked 'completed' (the poll handler records the payment), // otherwise 'failed'; a COMPLETED checkout releases the in-flight guard // (the poll handler records the payment); a definitively // cancelled/expired checkout is marked 'failed'; an ambiguous status is // left for a later run. // - till_sales.square_checkout_id: card-machine till sales. A checkout still // waiting at Square is cancelled and the sale marked 'failed' (the // payment_status enum has no 'cancelled' value, and 'failed' is the same // terminal state the stale-pending sweep uses, blocking the till // pending-retry path); a checkout that completed during the cancel window // (or was already COMPLETED when first checked) is RECORDED by the sweep — // the sale is marked 'completed' with the returned square_payment_id, the // till mirror of the booking path's recordUntrackedTerminalPayment (a // never-polled sale would otherwise stay invisible in till reporting until // the 24h blind-fail). // // Each row is checked at Square FIRST and only cancelled when the checkout is // provably still waiting (ErrCheckoutPending): a COMPLETED checkout is never // cancelled, and a checkout whose status is unknown (transport error) is left // alone for a later run. After a cancel the checkout is re-checked once — the // customer may have completed the payment in the cancel window, in which case // the row is resolved to 'completed' rather than 'failed'. func SweepStaleTerminalCheckouts(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-staleTerminalCheckoutAge) var pending []staleTerminalCheckoutRow // Booking terminal checkouts (CreateTerminalPayment) live in the // terminal_checkouts table. A stale PENDING/IN_PROGRESS row means the // checkout is still live at Square (or was left after a crash / lost poll). rows, err := db.Conn.Query(ctx, ` SELECT 'terminal_checkout', checkout_id, checkout_id, booking_id FROM terminal_checkouts WHERE status IN ('PENDING', 'IN_PROGRESS') AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() // Till-sale card-machine checkouts are tracked on the till_sales row. rows, err = db.Conn.Query(ctx, ` SELECT 'till_sale', id, square_checkout_id, '' FROM till_sales WHERE status = 'pending' AND square_checkout_id IS NOT NULL AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() resolved := 0 for _, r := range pending { // An EMPTY checkout_id is a legacy provisional (pre-Square) row for // which no Square call was ever possible — resolving it to failed // directly is safe (R3). if r.CheckoutID == "" { if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Provisional (pre-Square) terminal checkout row %s (%s) resolved as failed — no live checkout at Square", r.RowID, r.Kind) continue } // A provisional row carries a synthetic "tmp-" checkout_id. It is no // longer PROVABLY not live (H4): a hard crash between the // terminal_checkouts insert and the provisional→real UPDATE leaves a // LIVE checkout at Square (created under the idempotency key embedded // in the tmp id) while the row still carries the synthetic id. // Blind-failing would release the in-flight guard while that checkout // can still complete at the terminal into an untracked charge. Query // Square first and classify exactly like activeTerminalCheckoutID // (handlers.go H4): COMPLETED → record the payment and release the // guard; ErrCheckoutPending → the checkout is still live at Square, // keep the guard; isTerminalCheckoutError (NOT_FOUND / CANCELED) → // safe to fail (the crash happened before the Square call, or the // checkout was cancelled); ambiguous → leave the row in flight. A tmp- // id that never reached Square (or one Square cannot resolve) comes // back NOT_FOUND — the expected outcome — and failing it is correct. if strings.HasPrefix(r.CheckoutID, "tmp-") { pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case gErr == nil && pr.Status == "COMPLETED": // The checkout actually completed at the terminal. Record the // payment if it was never polled/recorded — a COMPLETED charge // must not stay an invisible untracked charge (H4). Booking // checkouts record full payment rows; till-sale checkouts // record the sale row. if r.Kind == "terminal_checkout" { if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) { resolved++ } } else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) { resolved++ } log.Printf("Provisional terminal checkout %s (%s %s) is COMPLETED at Square — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID) case errors.Is(gErr, square.ErrCheckoutPending): // The checkout is still live at Square — keep the in-flight // guard so a second checkout cannot be created while it can // still complete. log.Printf("Provisional terminal checkout %s (%s %s) is still live at Square — leaving pending; the in-flight guard stays held", r.CheckoutID, r.Kind, r.RowID) case isTerminalCheckoutError(gErr): // NOT_FOUND (no checkout was ever created — the crash happened // before the Square call) or CANCELED — safe to resolve failed. switch { case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr): if clawBackTillSaleFunding(ctx, r.RowID) { resolved++ } log.Printf("Provisional terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID) case r.Kind == "till_sale": if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("CRITICAL: provisional terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID) default: if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Provisional (pre-Square) terminal checkout row %q (%s %s) resolved as failed — no live checkout at Square", r.CheckoutID, r.Kind, r.RowID) } default: // Ambiguous error — the checkout's money state at Square is // unknown. Leave the row in flight rather than releasing the // guard and allowing a second checkout. log.Printf("Provisional terminal checkout %s (%s %s) status unknown (%v) — leaving pending for a later sweep", r.CheckoutID, r.Kind, r.RowID, gErr) } continue } // Conservative status check: only cancel a checkout that is provably // still waiting at Square. A COMPLETED checkout must never be // cancelled, and an ambiguous status (network error) is left alone for // the next run. pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case errors.Is(gErr, square.ErrCheckoutPending): // Still live at the terminal — cancel it. A customer can complete // the payment in the small window between the GetCheckout above and // the cancel, so re-check once before marking the row failed: a // COMPLETED charge must never be recorded as failed. if cErr := SquareClient.CancelCheckout(ctx, r.CheckoutID); cErr != nil { log.Printf("Failed to cancel stale terminal checkout %s (%s %s): %v", r.CheckoutID, r.Kind, r.RowID, cErr) continue } recheck, rErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case rErr == nil && recheck.Status == "COMPLETED": // The customer completed the payment during the cancel window. // Record it if it was never polled/recorded — a COMPLETED // checkout must not stay an untracked charge (H4). Booking // checkouts record full payments rows; till-sale checkouts // record the sale row (finding 3). if r.Kind == "terminal_checkout" { if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, recheck) { resolved++ } } else if recordUntrackedTillSalePayment(ctx, r.RowID, recheck) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) but re-check shows COMPLETED — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) case isTerminalCheckoutError(rErr) || errors.Is(rErr, square.ErrCheckoutPending): // The re-check after the cancel came back terminal-or-still-live // (CANCELED / NOT_FOUND / CANCEL_REQUESTED / PENDING). C3: the // clawback is gated on isCheckoutDefinitivelyDead exactly like // the first-check branch below — only an explicit CANCELED / // NOT_FOUND PROVES the charge can never complete. A checkout // still reporting live or cancel-requested at the re-check can // STILL complete at the terminal (the card payment raced the // cancel / Square's cancel propagation lagged), and clawing back // the funding on a charge that later lands would take the // customer's card money AND the gift-card value (silent // money-taken). switch { case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(rErr): // Provably dead — the funding has no charge behind it. if clawBackTillSaleFunding(ctx, r.RowID) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (till sale %s, pending >%s) — re-check proves the charge can never complete; marked failed with gift-card funding clawed back", r.CheckoutID, r.RowID, staleTerminalCheckoutAge) case !isCheckoutDefinitivelyDead(rErr): // The cancel did NOT provably take effect — the re-check // reports the checkout still live (PENDING / IN_PROGRESS / // CANCEL_REQUESTED). Marking the row failed now would DROP // the live checkout: a charge that completes at the terminal // LATER would land with no payments row and no alert — a // permanently untracked charge (money-F3). Keep the row in // flight for manual reconciliation (an operator verifies the // checkout state at Square), raise the CRITICAL admin // notification, and surface a till sale's outstanding // gift-card funding as an 'awaiting_reversal' trace row. leaveTerminalCheckoutReconciliationRequired(ctx, r) log.Printf("CRITICAL: stale terminal checkout %s (%s %s, pending >%s) was cancelled but the re-check is still live / not provably dead (%v) — NOT marked failed; left pending for manual reconciliation — verify the checkout state at Square whether the charge completed — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge, rErr) default: // Definitively terminal (CANCELED / NOT_FOUND) and no // gift-card funding to protect — safe to fail. if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) — marked failed", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) } default: // The cancel succeeded but the re-check itself is ambiguous — // leave the row for a later run. log.Printf("Terminal checkout %s was cancelled but its re-check is ambiguous (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, rErr, r.Kind, r.RowID) } case gErr == nil && pr.Status == "COMPLETED": if r.Kind == "terminal_checkout" { // The checkout completed at Square but was never polled/recorded // (the frontend never called GetCheckoutStatus). Record the // payment rows now — otherwise the charge stays invisible to // refunds and TotalPaid (H4). if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) { resolved++ } } else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) { // A never-polled COMPLETED till-sale checkout must not stay // pending until the 24h blind-fail (finding 3) — the charge is // real, so the sale is recorded completed with the payment id. resolved++ } case isTerminalCheckoutError(gErr): // The checkout is cancelled / cancel-requested / expired at Square. switch { case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr): // The error PROVES the checkout can never complete (CANCELED / // NOT_FOUND, and no bare CANCEL_REQUESTED) — claw back the // funded gift card along with the failed mark. if clawBackTillSaleFunding(ctx, r.RowID) { resolved++ } log.Printf("Terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID) case r.Kind == "till_sale": // CANCEL_REQUESTED-only: Square does not promise non-completion // in that state, so the charge may still land. Mark the sale // failed but DO NOT claw back the funded gift card. if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("CRITICAL: terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID) default: if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Terminal checkout %s is definitively terminal (%v) — marked %s %s failed", r.CheckoutID, gErr, r.Kind, r.RowID) } default: // Ambiguous transport/unknown status — leave for a later run. log.Printf("Terminal checkout %s status unknown (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, gErr, r.Kind, r.RowID) } } return resolved, nil } // staleTerminalCheckoutRow is one live-checkout row the sweep reads from // either table so it can resolve the checkout at Square before touching the // row. Kind is "terminal_checkout" (booking, terminal_checkouts table) or // "till_sale" (till_sales.square_checkout_id). BookingID is the booking the // terminal_checkouts row belongs to ("" for till sales) — needed to record an // untracked COMPLETED terminal charge (H4). type staleTerminalCheckoutRow struct { Kind string RowID string CheckoutID string BookingID string } // leaveTerminalCheckoutReconciliationRequired keeps a stale terminal checkout // row in flight (PENDING / IN_PROGRESS) after the sweep could NOT PROVABLY // cancel it at Square — the cancel-then-recheck still reports the checkout live // or only cancel-requested. Marking the row failed would DROP the live checkout: // a charge that later completes at the terminal would land with no payments row // and no admin alert — permanently untracked (money-F3). Instead the row stays // pending for manual reconciliation (an operator verifies the checkout state at // Square), the CRITICAL admin notification is raised (flood-capped, deduped per // booking/user), and a till sale's outstanding gift-card funding is surfaced as // an 'awaiting_reversal' trace row — the same manual-resolution path // insertOutstandingGiftCardFundingTrace drives — so the operator can see the // funding while reconciling. The row is deliberately NOT counted as resolved. func leaveTerminalCheckoutReconciliationRequired(ctx context.Context, r staleTerminalCheckoutRow) { var bookingID *string var userID *string if r.Kind == "terminal_checkout" && r.BookingID != "" { bookingID = &r.BookingID } if r.Kind == "till_sale" { // Attribute the alert to the funded gift card's redeemed-to user when // there is one and surface the outstanding funding as a trace row. var itemType string var itemID sql.NullString var totalAmount float64 var redeemedBy sql.NullString if err := db.Conn.QueryRow(ctx, ` SELECT ts.item_type, ts.item_id, ts.total_amount, gc.redeemed_by FROM till_sales ts LEFT JOIN gift_cards gc ON gc.id = ts.item_id WHERE ts.id = $1 `, r.RowID).Scan(&itemType, &itemID, &totalAmount, &redeemedBy); err != nil { log.Printf("CRITICAL: failed to look up till sale %s for outstanding-funding trace: %v — MANUAL RECONCILIATION REQUIRED", r.RowID, err) } else if itemType == "gift_card" && itemID.Valid && itemID.String != "" { if redeemedBy.Valid && redeemedBy.String != "" { userID = &redeemedBy.String } if _, tErr := db.Conn.Exec(ctx, ` INSERT INTO gift_card_transactions (gift_card_id, transaction_type, amount, reference_type, reference_id, user_id, notes) VALUES ($1, 'awaiting_reversal', $2, 'till_sale', $3, $4, $5) `, itemID.String, totalAmount, r.RowID, userID, "cancel-recheck still live: checkout not provably cancelled at Square — verify its state there manually before resolving"); tErr != nil { log.Printf("Failed to insert outstanding gift-card funding trace for till sale %s: %v", r.RowID, tErr) } } } insertCriticalPaymentNotification(ctx, bookingID, userID) } // markTerminalCheckoutRowFailed moves one tracked row to the terminal 'failed' // state after its checkout is cancelled or proven terminal at Square. Returns // true when the row was updated (status was still active). func markTerminalCheckoutRowFailed(ctx context.Context, r staleTerminalCheckoutRow) bool { var table, where string if r.Kind == "till_sale" { table = "till_sales" where = "id = $1 AND status = 'pending'" } else { table = "terminal_checkouts" where = "checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')" } // table is an internal constant ("till_sales"/"terminal_checkouts"), never // user input, but the identifier is routed through pgx.Identifier.Sanitize // so no raw, unquoted table name is ever concatenated into the statement. tag, err := db.Conn.Exec(ctx, ` UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW() WHERE `+where, r.RowID) if err != nil { log.Printf("Failed to mark %s %s failed: %v", r.Kind, r.RowID, err) return false } return int(tag.RowsAffected()) > 0 } // recordUntrackedTerminalPayment records the payments row(s) for a terminal // checkout that COMPLETED at Square but was never polled/recorded, then marks // the terminal_checkouts row COMPLETED. The stale sweep otherwise leaves a real // charge with NO payments row — invisible to refunds and TotalPaid (H4). The // money-recording work lives in the SHARED recordTerminalPaymentTx core (also // used by the GetCheckoutStatus poll handler), so both writers run the same // dedup / FOR UPDATE recheck / insert / M4 split / VAT / guarded checkout // update; this wrapper supplies the sweep-specific pieces: the advisory lock, // the cancelled-booking critical admin notification and the CRITICAL completion // log. Returns true when the checkout row was resolved (payment recorded, // already recorded, or refused on a cancelled booking); false when recording // failed (the row is left pending so the next sweep re-runs the whole // reconcile). func recordUntrackedTerminalPayment(ctx context.Context, checkoutID, bookingID string, pr *square.PaymentResult) bool { if pr == nil || pr.SquarePayID == "" { log.Printf("CRITICAL: terminal checkout %s is COMPLETED at Square but carries no Square payment ID — cannot record the payment — MANUAL RECONCILIATION REQUIRED", checkoutID) return false } // Serialize with the poll handler (GetCheckoutStatus): both record the same // Square payment, so the advisory lock + dedup SELECT prevent a double // insert (the idempotency_key UNIQUE constraint is the backstop). pinConn, err := db.Conn.Acquire(ctx) if err != nil { log.Printf("Failed to acquire connection for terminal-completion lock: %v", err) return false } defer pinConn.Release() lockOK, err := acquireAdvisoryLock(ctx, pinConn, "crussell:terminal:"+pr.SquarePayID) if err != nil { log.Printf("Failed to acquire terminal-completion serialization lock for %s: %v", pr.SquarePayID, err) return false } if !lockOK { log.Printf("Terminal-completion serialization lock for %s not acquired within bound — a poll is already recording this checkout", pr.SquarePayID) return false } defer releasePaymentLock(pinConn, "crussell:terminal:"+pr.SquarePayID) tx, err := db.Conn.Begin(ctx) if err != nil { log.Printf("Failed to begin terminal-completion transaction: %v", err) return false } defer func() { if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { log.Printf("Failed to rollback terminal-completion transaction: %v", err) } }() paymentID, recErr := recordTerminalPaymentTx(ctx, tx, checkoutID, bookingID, pr) if recErr != nil { if errors.Is(recErr, errTerminalBookingNotPayable) { // Sweep-only behaviour: the poll surfaces a 409 to a live caller, // but a sweep has no one to tell, so alert ops in the admin // notification centre that money was taken at Square and MUST be // refunded manually (the core already committed the checkout's // 'failed' mark). insertCriticalPaymentNotification(ctx, &bookingID, nil) return true } // The core logged the failure; the row is left pending so the next // sweep run re-runs the whole reconcile. return false } log.Printf("CRITICAL: recorded untracked terminal charge %s (booking %s) from stale checkout %s — payment row %s created (never polled by the frontend)", pr.SquarePayID, bookingID, checkoutID, paymentID) return true } // errTerminalBookingNotPayable is returned by recordTerminalPaymentTx when the // booking moved out of a payable state (cancelled / lapsed / no-show) between // the terminal charge completing at Square and the record attempt. The core has // ALREADY committed the terminal_checkouts 'failed' mark (and a non-payable // commit failure is wrapped with this sentinel too), so the caller must not // roll back: the poll surfaces a 409 conflict, the sweep inserts the critical // admin notification and resolves the row. var errTerminalBookingNotPayable = errors.New("booking is no longer payable for a terminal payment") // recordTerminalPaymentTx records a COMPLETED terminal checkout as payments // rows, applying the M4 tip split, per-record VAT and booking completion. Both // the GetCheckoutStatus poll handler and the stale-terminal sweep call it so // the money-recording logic exists exactly once. // // CALL-SITE CONTRACT: both call sites must keep passing the SAME inputs — the // caller's transaction, the terminal_checkouts checkout_id, the booking id and // the *square.PaymentResult returned by GetCheckout — and each must acquire the // advisory lock on "crussell:terminal:" on its own pinned pool // connection BEFORE calling (the core runs inside the caller's transaction). // The core commits the transaction and completes a now-fully-paid booking; the // caller owns only the error mapping (errTerminalBookingNotPayable vs generic), // the HTTP response / sweep return value and any post-commit behaviour (e.g. // the sweep's critical notification). func recordTerminalPaymentTx(ctx context.Context, tx pgx.Tx, checkoutID, bookingID string, pr *square.PaymentResult) (string, error) { service := NewPaymentService() // Dedup by Square payment ID: a concurrent poll (or a prior sweep run) // already recorded this charge — release the in-flight guard and return the // existing row so the caller reports the same payment id. The checkout is // marked COMPLETED under the same guarded status predicate used everywhere // else, so a row already resolved to 'failed' is never resurrected. var existingID string if err := tx.QueryRow(ctx, ` SELECT id FROM payments WHERE booking_id = $1 AND square_payment_id = $2 `, bookingID, pr.SquarePayID).Scan(&existingID); err == nil { if _, upErr := tx.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, checkoutID); upErr != nil { slog.Error("Failed to mark terminal checkout completed on dedup", "checkout_id", checkoutID, "err", upErr) return "", fmt.Errorf("mark terminal checkout %s completed: %w", checkoutID, upErr) } if cErr := tx.Commit(ctx); cErr != nil { slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", cErr) return "", cErr } return existingID, nil } else if !errors.Is(err, pgx.ErrNoRows) { slog.Error("Failed to check for existing terminal payment", "checkout_id", checkoutID, "booking_id", bookingID, "square_payment_id", pr.SquarePayID, "err", err) return "", err } // Re-check the booking status under the advisory lock (mirrors // GetCheckoutStatus): a cancellation/eviction that committed between the // terminal charge completing at Square and this record attempt must not // produce a completed payment on a cancelled/lapsed/no-show booking — the // cancellation refund path computes refunds from completed payments and // would silently exclude this charge. Mark the checkout failed and alert // ops: money was taken at Square and MUST be refunded manually. var recheckStatus string // FOR UPDATE (C5): serializes against the cancellation path's lock on the // same row so a concurrent cancellation cannot commit between this recheck // and the transaction commit below. if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, bookingID).Scan(&recheckStatus); err != nil { slog.Error("CRITICAL: Square payment was processed but re-reading booking status failed — manual reconciliation required", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "err", err) return "", err } if !bookingStatusAllowsCompletedPayment(recheckStatus) { slog.Error("CRITICAL: Square payment was processed but booking is no longer payable — marking checkout failed; money taken at Square MUST be refunded manually", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "status", recheckStatus) if _, upErr := tx.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'failed', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, checkoutID); upErr != nil { slog.Error("CRITICAL: Square payment landed on a non-payable booking but marking checkout failed errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", upErr) } if cErr := tx.Commit(ctx); cErr != nil { slog.Error("CRITICAL: Square payment landed on a non-payable booking and committing the checkout-failed mark errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", cErr) // Keep the sentinel wrapped so BOTH callers still surface the // conflict path (poll 409 / sweep notification) even when the // failed-mark commit itself failed. return "", fmt.Errorf("%w: %v", errTerminalBookingNotPayable, cErr) } return "", errTerminalBookingNotPayable } // The payment type the admin charged is recorded on the checkout row by // CreateTerminalPayment; fall back to 'full' for legacy rows. tip_enabled // records whether the customer EXPLICITLY requested a tip (the frontend's // tip_enabled flag): an overflow beyond the remaining booking value must // only be recorded as a tip record when the customer asked for one (B3) — // an accidental overpayment must never be relabelled gratuity. var checkoutPaymentType string var checkoutTipEnabled bool if err := tx.QueryRow(ctx, ` SELECT payment_type, tip_enabled FROM terminal_checkouts WHERE checkout_id = $1 `, checkoutID).Scan(&checkoutPaymentType, &checkoutTipEnabled); err != nil { if !errors.Is(err, pgx.ErrNoRows) { slog.Error("Failed to read payment type for checkout", "checkout_id", checkoutID, "err", err) } checkoutPaymentType = "full" } idempotencyKey := bookingID + "-terminal-" + strconv.FormatInt(pr.Amount, 10) + "-" + pr.SquarePayID record := PaymentRecord{ BookingID: bookingID, PaymentType: checkoutPaymentType, PaymentMethod: "in_person_card", Status: "completed", Amount: float64(pr.Amount) / 100.0, SquarePaymentID: &pr.SquarePayID, IdempotencyKey: &idempotencyKey, Fees: float64(pr.Fees) / 100.0, CreatedAt: clock.Now(), UpdatedAt: clock.Now(), } // The booking user is read here (before the M4 carve) because the carve // computes the campaign discounts applyEligibleCampaignsAtPayment is about // to apply below, and that apply needs the payer's user id. var bookingUserID string if err := tx.QueryRow(ctx, `SELECT COALESCE(user_id, '') FROM bookings WHERE id = $1`, bookingID).Scan(&bookingUserID); err != nil { slog.Error("Failed to load booking user for terminal campaign apply", "booking_id", bookingID, "err", err) } // M4 split: a terminal charge above the remaining booking value is a tip — // record it as its own record so only the booking portion is refundable. // B3: only when the customer EXPLICITLY requested a tip (checkout // tip_enabled). Without an explicit tip request the overflow is an // accidental overpayment, not gratuity: it stays on the booking record at // the full charged amount (the excess is refundable, never a tip). The // checkout-creation side clamps the amount to the remaining value when no // tip is requested, so this branch only fires for legacy/edge checkouts. var records []PaymentRecord bookingInfo, bErr := service.GetBookingPaymentInfo(ctx, bookingID) if bErr == nil && bookingInfo != nil { charged := float64(pr.Amount) / 100.0 remainingBookingValue := math.Max(0, bookingInfo.TotalAmount-bookingInfo.TotalPaid) // Reserve the headroom of the campaign discounts that // applyEligibleCampaignsAtPayment is about to mint: a // terminal charge already priced to the discounted amount (the frontend // charges total−discount plus any explicit tip) must carve the tip // against the DISCOUNTED obligation, not the full total — otherwise the // whole charge lands as booking portion and the tip is silently // absorbed into deposit/balance instead of being recorded as gratuity // (M4). The no-tip case is unchanged: the booking portion exactly // covers the discounted obligation and the discount fills the rest. if pending := pendingCampaignDiscountAmount(ctx, tx, bookingID, bookingUserID, bookingInfo.TotalAmount); pending > 0.004 { remainingBookingValue = math.Max(0, remainingBookingValue-pending) } bookingPortion := math.Min(charged, remainingBookingValue) bookingPortion = math.Round(bookingPortion*100) / 100 tipAmount := math.Max(0, math.Round((charged-bookingPortion)*100)/100) if checkoutTipEnabled && tipAmount > 0.004 { records = buildTerminalSplitRecords(record, bookingInfo, bookingPortion, tipAmount) } } hasTip := false for _, rec := range records { if rec.PaymentType == "tip" { hasTip = true break } } // Apply eligible campaigns BEFORE the records are inserted when the M4 // carve minted a TIP record and a campaign discount is pending. The apply // path refuses discounts once a booking has 2+ completed real payments, and // the split records (deposit + balance + tip) would count as 2+ even though // they are ONE charge — refusing the discount the frontend already priced // in would leave the booking under-discounted and never complete. Applying // before the insert is money-safe: a tip record only exists when the charge // exceeds the DISCOUNTED obligation, so the booking portion plus the // pending discount can never exceed the total — the F1 full-amount // over-credit cannot occur here (a full-amount charge never produces a tip // record, so it keeps the after-insert apply whose cap refuses it). appliedCampaignsBeforeInsert := false if hasTip && bookingInfo != nil && bErr == nil { if pendingCampaignDiscountAmount(ctx, tx, bookingID, bookingUserID, bookingInfo.TotalAmount) > 0.004 { if applyErr := applyEligibleCampaignsAtPayment(ctx, tx, bookingID, bookingUserID, nil); applyErr != nil { slog.Error("Failed to apply eligible campaigns before terminal payment insert", "booking_id", bookingID, "err", applyErr) } appliedCampaignsBeforeInsert = true } } if len(records) == 0 { records = []PaymentRecord{record} } primary := records[0] paymentID, err := service.CreatePaymentRecordTx(ctx, tx, primary, nil) if err != nil { slog.Error("Failed to create payment record for terminal charge", "square_payment_id", pr.SquarePayID, "err", err) return "", err } ApplyVATToBookingPayment(ctx, tx, paymentID) for _, rec := range records[1:] { pid, cErr := service.CreatePaymentRecordTx(ctx, tx, rec, nil) if cErr != nil { slog.Error("Failed to create terminal tip split record", "square_payment_id", pr.SquarePayID, "err", cErr) return "", cErr } ApplyVATToBookingPayment(ctx, tx, pid) } // B13: apply eligible campaign discounts at record time, inside the same // transaction as the payment insert. The card-machine checkout completes // ASYNCHRONOUSLY (the GetCheckoutStatus poll or the stale-checkout sweep), // so there is no HTTP response to surface an exhausted-at-apply campaign // the way the synchronous saved-card/booking paths do — a campaign that // exhausts between the frontend's preview and this record stays a silent // skip (the customer was already charged the discounted amount the frontend // showed; the merchant absorbs the shortfall — documented limitation of the // async flow). An AVAILABLE campaign MUST be applied here rather than // deferred to the completion side-effects: completeFullyPaidBooking only // runs ApplyBookingCompletionSideEffects when bookingIsFullyPaid (real // money covers the total), at which point capDiscountToRemainingObligation // sees zero headroom — so a discounted-amount terminal charge would never // get its discount row and never complete. Applying here (with the same // capDiscountToRemainingObligation guard as every other path) mints the // discount row so bookingIsFullyPaid sees it; the completion side-effects // skip re-application via their already-recorded guards. The apply runs // AFTER the payment record insert so the headroom counts the charge as real // money (never over-credits), and after the M4 tip split so tip records // never affect the discount computation. When the M4 carve minted a tip // record the discount was already applied BEFORE the insert (see above), so // the apply is skipped here — a second run is idempotent but unnecessary. // The booking user was already read above for the M4 carve. if !appliedCampaignsBeforeInsert { if applyErr := applyEligibleCampaignsAtPayment(ctx, tx, bookingID, bookingUserID, nil); applyErr != nil { slog.Error("Failed to apply eligible campaigns for terminal checkout", "booking_id", bookingID, "err", applyErr) } } // Release the in-flight guard: this checkout is now recorded. if _, err := tx.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, checkoutID); err != nil { slog.Error("Failed to mark terminal checkout completed", "checkout_id", checkoutID, "err", err) return "", err } if err := tx.Commit(ctx); err != nil { slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", err) return "", err } // The booking may now be fully paid — complete it like the poll handler does. completeFullyPaidBooking(ctx, bookingID) return paymentID, nil } // pendingCampaignDiscountAmount sums the campaign discounts that // applyEligibleCampaignsAtPayment is about to apply for the booking (its // 'expected' argument is nil in the terminal flow, so it recomputes the // eligible set from scratch). The discount rows do not exist yet when the M4 // tip carve runs — they are minted later in the same transaction — so the // carve must reserve their headroom NOW or a terminal charge already priced to // the discounted amount absorbs the customer's explicit tip into the booking // portion (M4). Referral discounts are excluded: the terminal apply path never // mints them. func pendingCampaignDiscountAmount(ctx context.Context, q db.Querier, bookingID, userID string, bookingTotal float64) float64 { var total float64 for _, d := range ComputeEligibleDiscounts(ctx, q, bookingID, userID, bookingTotal) { if d.Source == "campaign" { total += d.Amount } } return math.Round(total*100) / 100 } // recordUntrackedTillSalePayment records a stale card-machine till sale whose // checkout COMPLETED at Square but was never polled/recorded: the sale is // marked 'completed' with the returned square_payment_id written back. A // never-polled COMPLETED checkout would otherwise leave the sale pending until // the 24h blind-fail — the charge is real but invisible in till reporting // (money-safe: the gift card was funded pre-charge, a blind-fail never claws // back, and failed rows reject retries — but the till reporting would be wrong, // finding 3). There is no split/deposit concept for till sales — a single // completed row, exactly the update the till poll handler (GetTillCheckoutStatus) // performs. The WHERE status = 'pending' guard makes it safe against a // concurrent poll: only one of them wins the update, the loser sees 0 rows and // backs off. Returns true when the sale was updated; false when the checkout // carries no Square payment id (the sale is left PENDING — never blind-failed // — with a CRITICAL log for manual reconciliation) or the sale was already // resolved by someone else. func recordUntrackedTillSalePayment(ctx context.Context, saleID string, pr *square.PaymentResult) bool { if pr == nil || pr.SquarePayID == "" { log.Printf("CRITICAL: terminal checkout is COMPLETED at Square but carries no Square payment ID — cannot record till sale %s — leaving it PENDING — MANUAL RECONCILIATION REQUIRED", saleID) return false } tag, err := db.Conn.Exec(ctx, ` UPDATE till_sales SET status = 'completed', square_payment_id = $1, updated_at = NOW() WHERE id = $2 AND status = 'pending' `, pr.SquarePayID, saleID) if err != nil { log.Printf("Failed to record untracked terminal till sale %s as completed: %v", saleID, err) return false } if int(tag.RowsAffected()) == 0 { // A concurrent poll (or a prior sweep run) already recorded the sale — // nothing left for this run to do. log.Printf("Terminal till sale %s was already resolved — skipping untracked terminal completion record", saleID) return false } // FIX 2: the synchronous till-completion path (GetTillCheckoutStatus) and // the stale-pending rescue apply VAT via ApplyVATToTillSale; an untracked // terminal till sale recorded here must do the same or the sale silently // drops out of VAT reporting. The SQL function is idempotent (guarded on // vat_amount IS NULL), so a run racing the poll's own apply is a no-op. ApplyVATToTillSale(ctx, db.Conn, saleID) log.Printf("CRITICAL: recorded untracked terminal till-sale charge %s (sale %s) from a stale checkout — sale marked completed (never polled by the frontend)", pr.SquarePayID, saleID) return true } // isTerminalCheckoutError reports whether a GetCheckout error proves the // checkout can never complete. Square's HTTP client returns ErrCheckoutPending // for a still-live checkout and surfaces a definitively CANCELED status as a // "square: checkout is CANCELED (not COMPLETED)" error; an expired // checkout returns a structured NOT_FOUND API error (the mock uses a plain // "checkout not found"). Any other error (timeout, 5xx) leaves the money state // ambiguous, so the checkout must stay in flight. // // Structured codes are authoritative when present: NOT_FOUND (via // square.IsNotFound) and an explicit CANCELED / CANCEL_REQUESTED code classify // as terminal. The formatted-message match applies only to errors that carry // no structured code — the dev mock's plain errors and the real client's // client-side "is (not COMPLETED)" error, which it synthesizes // without a squareAPIError. func isTerminalCheckoutError(err error) bool { if err == nil || errors.Is(err, square.ErrCheckoutPending) { return false } if square.IsNotFound(err) || squareHasCode(err, "CANCELED", "CANCEL_REQUESTED") { return true } // Any other structured Square error code is authoritative — never // substring-match its message. if square.ErrorCode(err) != "" { return false } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "CANCELED") || strings.Contains(msg, "CANCEL_REQUESTED") || strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") } // isCheckoutDefinitivelyDead reports whether a GetCheckout error PROVES the // checkout can never complete — the condition under which a till sale's funded // gift card may be clawed back. It is stricter than isTerminalCheckoutError: // a CANCEL_REQUESTED-only classification (Square does not promise non-completion // in that state) is NOT definitive proof, so the charge may still land and the // funding must stay put. Only an explicit CANCELED or NOT_FOUND status is // definitive; a CANCEL_REQUESTED message counts only when it ALSO carries // CANCELED. // // Structured codes are authoritative when present: CANCELED and NOT_FOUND are // definitive, a bare CANCEL_REQUESTED code is not. The formatted-message match // applies only to errors that carry no structured code (dev mock plain errors, // and the real client's client-side "is (not COMPLETED)" error). func isCheckoutDefinitivelyDead(err error) bool { if err == nil || errors.Is(err, square.ErrCheckoutPending) { return false } if code := square.ErrorCode(err); code != "" { return code == "CANCELED" || code == "NOT_FOUND" } // Non-structured errors: an HTTP 404 / plain 404 body is definitive // (IsNotFound); the message match covers the client-side CANCELED status // error and the mock's plain not-found errors. if square.IsNotFound(err) { return true } msg := strings.ToUpper(err.Error()) if !strings.Contains(msg, "CANCELED") && !strings.Contains(msg, "NOT_FOUND") && !strings.Contains(msg, "NOT FOUND") { return false } if strings.Contains(msg, "CANCEL_REQUESTED") && !strings.Contains(msg, "CANCELED") { return false } return true } // clawBackTillSaleFunding reverts the gift-card funding of a till sale whose // card-machine checkout is provably dead, marking the sale failed at the same // time. The terminal-checkout sweep's rows carry no gift-card context, so the // sale's item/amount/redeem target are looked up here (is_create via the // created_at equality with the gift card) and handed to the claim-first // revertGiftCardFunding. A sale with no gift card (future retail product / // orphaned item) is only marked failed. Returns true when the sale was // resolved to failed; false when it was already resolved by someone else, had // no gift card, or the clawback failed (CRITICAL logged). func clawBackTillSaleFunding(ctx context.Context, tillSaleID string) bool { var itemType string var itemID sql.NullString var totalAmount float64 var redeemedBy sql.NullString var isCreate *bool err := db.Conn.QueryRow(ctx, ` SELECT ts.item_type, ts.item_id, ts.total_amount, gc.redeemed_by, (ts.created_at = gc.created_at) AS is_create FROM till_sales ts LEFT JOIN gift_cards gc ON gc.id = ts.item_id WHERE ts.id = $1 `, tillSaleID).Scan(&itemType, &itemID, &totalAmount, &redeemedBy, &isCreate) if err != nil { log.Printf("CRITICAL: failed to look up till sale %s for funding clawback: %v — MANUAL RECONCILIATION REQUIRED", tillSaleID, err) return false } if itemType != "gift_card" || !itemID.Valid || itemID.String == "" || isCreate == nil { // No gift card to claw back — mark the sale failed without touching // any card. tag, upErr := db.Conn.Exec(ctx, ` UPDATE till_sales SET status = 'failed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, tillSaleID) if upErr != nil { log.Printf("Failed to mark till sale %s failed: %v", tillSaleID, upErr) return false } return int(tag.RowsAffected()) > 0 } action := "topup" if *isCreate { action = "create" } var redeem *string if redeemedBy.Valid && redeemedBy.String != "" { redeem = &redeemedBy.String } if revErr := revertGiftCardFunding(ctx, action, itemID.String, totalAmount, redeem, tillSaleID); revErr != nil { if errors.Is(revErr, errTillSaleNotPending) { log.Printf("Till sale %s was already resolved (not pending) — skipping funding clawback", tillSaleID) return false } log.Printf("CRITICAL: funding clawback for till sale %s (gift card %s) failed: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", tillSaleID, itemID.String, revErr) return false } return true }