Files
Crussell/backend/handlers/payments/sweep.go
T
popertots 3866cc5963 fix: round-2 loop-A fresh review (503c326 baseline) — B1 replay cap, A6 discount record, 2FA reissue+cooldown, notification flood, lockout saturation, VAT/refund-status consolidation
Round 2 Loop A fresh money/security/dup-mod review. 23 findings fixed:

MONEY:
- CRITICAL: B1 duplicate auto-refund gains an attempt cap (b1_attempts col, cap 3) —
  a rejected auto-refund no longer re-replays the expired key every sweep run
  (which minted a stacking unauthorized charge each time); FAILED-webhook
  demotion respects the cap; never re-replay a key whose B1 refund failed
- HIGH: A6 deposit_covered_by_discount skip path now APPLIES the eligible
  campaign discount rows immediately (capped) instead of skipping with no
  discount recorded — no more promised-discount-not-recorded overcharge
- MEDIUM: 2FA code burned by the SAVE gate is re-issued on failed
  new-card+save_card charges (re-issue guard now covers req.SaveCard)
- LOW: GetBookingPaymentSummary excludes tip rows from paidAmount (remaining
  now matches the authoritative tip-excluded balance)

SECURITY:
- MEDIUM: unacknowledged CRITICAL admin-notification flood capped (global cap
  on critical_payment_log + refresh_token_reuse rows)
- MEDIUM: 2FA reissue no longer bypasses the mint cooldown (Check no longer
  clears LastMintAt on gate-verify; cleared on terminal charge success)
- MEDIUM: twofa.StateFor map-saturation returns a shared permanently-locked
  state instead of a fresh 5-guess budget per request
- MEDIUM: ProgressiveRateLimit rejects 429 past maxProgressiveSleepDelayMs
  instead of sleeping unboundedly; login bcrypt concurrency semaphore added
- LOW: loginInProgress 409->429; webhook key-set/URL-unset startup check;
  email-verification per-user attempt counter

DUP/MOD:
- formatCurrency single source (frontend format.ts, 7 files consolidated);
  SquareRefundStatusToLocal single source (errors.go, all sites); admin
  audit-log helper dedup; SCA retry model unified (proactive on all 6
  surfaces); buyDailyTotal/daily-cap mirror via backend; lock TTL from
  backend; generateUUID at all card-form sites; magic numbers named
  (defaultPostgresHost, epsilon, fee constants); admin CASH + gift-card
  terminal charges now audited; DAV_SKIP_INIT documented in manuals

Verified: 26/26 dev + 24/24 prod (GO_TESTING=1, the CI condition), both vet
tags, frontend tests+build, env-docs 42/42.
2026-08-22 00:34:50 +01:00

2566 lines
133 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package payments
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log"
"log/slog"
"math"
"strconv"
"strings"
"time"
"crussell/clock"
"crussell/db"
"crussell/internal/square"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
// SweepStalePendingPayments resolves pending payment records that are older
// than Square's idempotency-key retention window (~24h). A pending record
// means the DB committed but the Square charge outcome is unknown; it
// normally resolves on a same-key client retry. But if the client abandoned
// the attempt, the record stays pending forever — and retrying it after the
// key expires would ISSUE A SECOND CHARGE (Square no longer dedups). Failing
// stale pendings closes that double-charge window: a late retry finds a
// 'failed' record and stops instead of charging again.
//
// Before failing a row, the sweep reconciles it against Square: a
// genuinely-charged row (Square success, DB post-charge failure) with a
// square_payment_id is rescued to 'completed' instead of being swept to
// 'failed' with no automatic resolution — the money would otherwise be lost
// in limbo (MINOR-R3). Reconciliation is deliberately minimal: status +
// updated_at only, no split/VAT recomputation (that is the handler's job; the
// row is >24h stale and this is a reconciliation rescue). A row with NO
// square_payment_id but a STORED idempotency key (the lost-response case — the
// charge may have completed at Square with the response never received) is
// reconciled at Square by replaying the key (ReplayPaymentByKey) at an EARLIER
// 22h cutoff, while the key is still inside Square's ~24h retention window: a
// COMPLETED charge is rescued to 'completed' with the real square_payment_id
// written back, and a charge Square proves never happened is failed. Rows with
// neither a square_payment_id nor a stored key cannot be reconciled and are
// failed with a WARN exactly as the legacy sweep did.
//
// Only online/till card payments can be pending — cash/giftcard/on_the_house
// are committed synchronously and never enter this state. Both the payments
// table and till_sales carry pending card-sale rows and are swept here.
const stalePendingPaymentAge = 24 * time.Hour
// stalePendingKeyedAge is how old a pending row with a stored idempotency key
// (but no square_payment_id — the lost-response case) must be before the sweep
// reconciles it at Square by replaying the key. It is deliberately 2h EARLIER
// than stalePendingPaymentAge so the replay lands comfortably inside Square's
// ~24h idempotency-key retention window: replaying at exactly 24h risks the key
// already having expired, and an expired key would make the probe rejection
// look like "never charged" even when the charge actually landed. Rows older
// than stalePendingPaymentAge when swept can no longer be replayed
// trustworthily and fall back to the legacy blind-fail + WARN.
const stalePendingKeyedAge = 22 * time.Hour
// b1DuplicateRefundAttemptCap caps how many times the sweep may auto-refund a
// replay-induced duplicate charge (B1) under one stale pending row's expired
// idempotency key. A REJECTED refund writes NO refunds row, so the in-flight
// guard (hasInFlightSweepDuplicateRefund) stays false and the next run would
// re-replay the SAME expired key — Square mints ANOTHER charge, the refund is
// rejected again, and the loop stacks unauthorized charges over the row's 24h
// life. The cap stops the loop: a row at the cap is failed with a CRITICAL
// notification and its key is never replayed. Mirrors maxManualRefundAttempts
// (refunds.go).
const b1DuplicateRefundAttemptCap = 3
// SweepStalePendingPayments resolves stale pending payments and till sales; see the rationale block on stalePendingPaymentAge above.
func SweepStalePendingPayments(ctx context.Context) (int, error) {
cutoff := clock.Now().Add(-stalePendingPaymentAge)
keyedCutoff := clock.Now().Add(-stalePendingKeyedAge)
// Pass 0: stranded Square-less MANUAL refund rows at the attempt cap
// (maxManualRefundAttempts). The Square-less pre-pass inside
// sweepManualPendingSquareRefunds (refunds.go) reconciles only rows with
// refund_attempts < maxManualRefundAttempts; legacy rows
// that already hit the cap are never reconciled by it and would stay
// 'pending' forever, permanently blocking the over-refund guard. Such rows
// can never be refunded via Square (no square_payment_id), so they are
// marked 'failed' + admin-notified here for in-person arrangement (F10).
sqlessRefundCount, sqlessErr := sweepSquarelessManualRefundsAtAttemptCap(ctx)
if sqlessErr != nil {
log.Printf("Failed to reconcile Square-less manual refunds at the 3-attempt cap: %v", sqlessErr)
}
// Pass 1 (earlier cutoff): rows with a stored idempotency key but NO
// square_payment_id are the lost-response case — the charge may have landed
// at Square with the response lost. They are reconciled at Square by
// replaying the key (ReplayPaymentByKey) while the key is still inside
// Square's ~24h retention window: a COMPLETED charge is rescued to
// 'completed' with the real square_payment_id, a charge Square proves never
// happened is failed (and a till sale's funded gift card clawed back), an
// ambiguous answer leaves the row pending for the next run, and a row
// already past the retention window is blind-failed with a WARN exactly as
// the legacy sweep did (replaying an expired key would misread the probe
// rejection as "never charged").
payKeyedCount, payKeyedCompleted, payKeyedUnverified, err := sweepKeyedStaleRows(ctx, "payments", keyedCutoff)
if err != nil {
return 0, err
}
tillKeyedCount, tillKeyedCompleted, tillKeyedUnverified, err := sweepKeyedStaleRows(ctx, "till_sales", keyedCutoff)
if err != nil {
return 0, err
}
// Pass 2 (legacy 24h cutoff): rows WITH a square_payment_id are reconciled
// by payment id; rows with neither a payment id nor a stored key cannot be
// reconciled and are failed directly (WARN — the charge outcome is unknown).
payCount, payCompleted, err := sweepStaleRows(ctx, "payments", cutoff)
if err != nil {
return 0, err
}
tillCount, tillCompleted, err := sweepStaleRows(ctx, "till_sales", cutoff)
if err != nil {
return 0, err
}
total := sqlessRefundCount + payKeyedCount + tillKeyedCount + payCount + tillCount
payTotal := payKeyedCount + payCount
tillTotal := tillKeyedCount + tillCount
completed := payKeyedCompleted + tillKeyedCompleted + payCompleted + tillCompleted
if total > 0 {
log.Printf("[SWEEP] Resolved %d stale pending payments (%d payments, %d till sales) older than %s — late retries will be rejected, preventing a second Square charge; %d reconciled to completed against Square (%d payments, %d till sales)", total, payTotal, tillTotal, stalePendingPaymentAge, completed, payKeyedCompleted+payCompleted, tillKeyedCompleted+tillCompleted)
}
// Routine sweep bookkeeping, not an incident: failing stale pending rows is
// the sweep's DESIGNED behaviour. A row swept to failed may still have been
// charged at Square with a lost response, so note it at WARN level for an
// admin doing a periodic money reconciliation — but this fires on every
// normal run and must not be elevated to CRITICAL (which is reserved for
// genuinely unrecoverable post-charge branches). Rows the keyed reconcile
// PROVED never charged are deliberately excluded from the WARN (they cannot
// have moved money); only rows failed without a reconcile proof — keyless
// rows and keyed rows past the retention window — are counted.
if payKeyedUnverified > 0 {
log.Printf("[SWEEP] WARN: %d pending payments with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response — verify before refunding/charging", payKeyedUnverified)
}
if tillKeyedUnverified > 0 {
log.Printf("[SWEEP] WARN: %d pending till sales with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response", tillKeyedUnverified)
}
if payCount > 0 {
log.Printf("[SWEEP] WARN: %d pending payments marked failed may have been charged at Square with a lost response — verify before refunding/charging", payCount-payCompleted)
}
if tillCount > 0 {
log.Printf("[SWEEP] WARN: %d pending till sales marked failed may have been charged at Square with a lost response", tillCount-tillCompleted)
}
return total, nil
}
// staleRow is one stale pending row read by the sweep so it can reconcile
// rows that carry a Square reference BEFORE failing them. For till_sales rows
// the gift-card context needed to claw back funded money on a definitive
// failure is carried alongside: the card id, who it was redeemed to, whether
// this sale created the card (created_at equality — provable because both
// timestamps are the transaction-start NOW() in the same tx), and the sale
// amount (the exact funding this sale added).
type staleRow struct {
ID string
SquarePaymentID string
// IdempotencyKey is the deterministic Square idempotency key stored on the
// pending row ("" when absent). Rows WITH a key but NO square_payment_id are
// the lost-response case: the sweep replays the key at Square to learn the
// true charge outcome before declaring failure.
IdempotencyKey string
// SquareSourceID is the row's CURRENT square_source_id — the source_id sent
// in the latest CreatePayment attempt, refreshed whenever a same-key retry
// reuses the pending row. The replay-by-key overrides the stored snapshot's
// embedded source with this value so the replayed body matches the source
// the retained key actually used. Replaying a different body would return
// IDEMPOTENCY_KEY_REUSED, which proves nothing about whether the charge
// landed.
SquareSourceID string
// SquareRequestSnapshot is the verbatim original CreatePayment request JSON
// stored on the row at charge time (square_request_snapshot) — the FULL
// body the replay-by-key must repeat (source, key, amount, customer_id,
// reference_id, note, buyer_email_address, verification_token, ...).
// Square's idempotency dedup compares the whole request, so a replay built
// from partial row data returns IDEMPOTENCY_KEY_REUSED for a RETAINED key
// and the row stays pending forever (safe but never auto-rescued). Nil for
// legacy rows — the sweep then rebuilds the minimal body from the stored
// key + source + amount.
SquareRequestSnapshot []byte
// AmountPence is the row's charge amount in pence — the amount the original
// CreatePayment used. The replay-by-key must repeat it so Square's
// idempotency dedup returns the original payment.
AmountPence int64
// CreatedAt is the pending row's creation time; rows already past Square's
// idempotency-key retention window cannot be replayed trustworthily.
CreatedAt time.Time
// BookingID is the payments row's booking_id ("" when the row has none).
// A payments row with NO booking is a gift-card purchase (BuyGiftCard
// inserts without a booking): rescuing it to 'completed' on a Square
// COMPLETED reconcile would permanently block the same-key retry that
// delivers the card, so such rows are kept pending instead (C6).
BookingID *string
// CreatedBy is the payments row's created_by user id (gift-card purchases
// always carry the purchaser), used to attribute the critical-payment admin
// notification.
CreatedBy *string
ItemID string // till_sales.item_id — the gift card ("" when NULL / non-gift-card)
RedeemToUserID *string // gift_cards.redeemed_by — user credited by a create-with-redeem
IsCreate bool // true when this sale created the gift card (timestamps equal)
HasGiftCard bool // false when the LEFT JOIN found no gift_cards row (gc.id IS NULL)
TotalAmount float64 // till_sales.total_amount — the funding this sale added
// B1Attempts counts how many times the sweep has auto-refunded a
// replay-induced duplicate charge under this row's expired idempotency key.
// A REJECTED refund (or the cap) means the key must never be replayed again.
B1Attempts int
}
// sweepStaleRows resolves the stale pending rows of one table. Rows with a
// square_payment_id are reconciled at Square first (COMPLETED → 'completed',
// anything else → 'failed' exactly as the legacy bulk UPDATE did); rows
// without one cannot be reconciled and are failed directly. A till_sales row
// whose reconcile PROVES the charge never completed (NOT_FOUND / non-COMPLETED)
// also claws back the funded gift card atomically with the failed mark; the
// blind-fail path (no square_payment_id — the charge may have landed) never
// claws back. Returns the total rows resolved and how many were rescued to
// 'completed'.
func sweepStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved int, completed int, err error) {
switch table {
case "payments", "till_sales":
default:
return 0, 0, fmt.Errorf("sweep: unknown stale table %q", table)
}
stale, err := fetchStaleRows(ctx, table, cutoff, false)
if err != nil {
return 0, 0, err
}
for _, r := range stale {
// B1: a row whose auto-refund of a replay-induced duplicate charge is
// still PENDING at Square must never be blind-failed (or have its gift
// card clawed back) — the refund is non-terminal and may still settle.
// The B1 re-poll pass resolves it on settlement.
if hasInFlightSweepDuplicateRefund(ctx, table, r.ID) {
log.Printf("Stale pending %s row %s has a sweep auto-refund of a duplicate charge still pending at Square — leaving pending until the refund settles", table, r.ID)
continue
}
if r.SquarePaymentID != "" {
switch reconcileStalePaymentAtSquare(ctx, table, r.SquarePaymentID) {
case staleReconcileCompleted:
if table == "payments" && r.BookingID == nil {
// Gift-card purchase row: the charge landed at Square but
// the gift card was never delivered (C6). Completing the row
// permanently blocks the same-key retry that delivers the
// card — leave it pending and alert.
leaveGiftCardPurchasePending(ctx, r)
continue
}
// F3: a charge Square reports COMPLETED must never be
// completed on a booking that was cancelled during the pending
// window — the cancellation refund path computes refunds from
// completed payments and would miss it, charging the customer
// with NO automatic refund. Re-read bookings.status (FOR
// UPDATE) and refuse on a cancelled booking. Rows with no
// booking (gift-card purchases, till_sales) are never gated.
if table == "payments" {
switch gateStalePaymentRescueOnBooking(ctx, r) {
case staleBookingGateRefused:
resolved++
continue
case staleBookingGateUnknown:
continue
}
}
if rescueStaleRowCompleted(ctx, table, r) {
resolved++
completed++
}
continue
case staleReconcileLeavePending:
// Square's answer was ambiguous (transport/5xx) — the charge
// may still be in flight at Square. Do NOT touch the row: the
// next sweep run reconciles it again, and a same-key retry
// must still be able to reuse the pending row if the charge
// actually completed.
log.Printf("Stale pending %s row %s left pending (Square reconcile ambiguous) — will retry next sweep", table, r.ID)
continue
}
// staleReconcileDefinitivelyFailed falls through to the fail path.
}
// Fail path. A till-sale reconcile that PROVED the charge never
// completed (NOT_FOUND / non-COMPLETED) claws back the funded gift
// card atomically with the failed mark (HIGH-1). The blind-fail path
// (no square_payment_id — the charge outcome is unknown) NEVER claws
// back: the money may have landed at Square.
if table == "till_sales" && r.SquarePaymentID != "" && r.HasGiftCard {
if clawbackTillSaleFunding(ctx, r) {
resolved++
}
continue
}
if failStaleRow(ctx, table, r.ID) {
resolved++
// A5a: blind-failing a row with NO square_payment_id leaves the
// charge outcome unknown (the money may have landed at Square with a
// lost response), so surface it in the admin notification centre.
// Rows that reach this line WITH a square_payment_id were PROVED
// never-charged by the by-id reconcile and need no alert.
if r.SquarePaymentID == "" {
notifyStaleRowCritical(ctx, r)
}
}
}
return resolved, completed, nil
}
// sweepKeyedStaleRows resolves stale pending rows that carry a stored
// idempotency key but no square_payment_id — the lost-response case, where the
// charge may have completed at Square with the response never reaching the
// app. Such rows are reconciled at Square by replaying the key while it is
// still inside Square's ~24h retention window (the earlier stalePendingKeyedAge
// cutoff guarantees this):
//
// - a COMPLETED payment under the key → rescued to 'completed' with the real
// square_payment_id written back (the customer WAS charged);
// - a payment Square proves never happened (unknown key / FAILED status) →
// marked 'failed', and a till sale's funded gift card is clawed back
// (reconcile PROVED the funding has no charge behind it);
// - an ambiguous answer (transport/5xx) → left pending for the next run;
// - a row already older than the retention window when swept → cannot be
// replayed trustworthily (an expired key would misread the probe rejection
// as "never charged" even when the charge landed), so it is blind-failed
// with a WARN exactly as the legacy sweep did.
//
// Returns the total rows resolved, how many were rescued to 'completed', and
// how many were marked failed WITHOUT a reconcile proof (blind-fail WARNs).
func sweepKeyedStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved, completed, unverifiable int, err error) {
switch table {
case "payments", "till_sales":
default:
return 0, 0, 0, fmt.Errorf("sweep: unknown stale table %q", table)
}
stale, err := fetchStaleRows(ctx, table, cutoff, true)
if err != nil {
return 0, 0, 0, err
}
// Square's idempotency-key retention window closes at stalePendingPaymentAge;
// a row older than that when swept can no longer be replayed trustworthily.
replayExpired := clock.Now().Add(-stalePendingPaymentAge)
for _, r := range stale {
// B1: a row whose auto-refund of a replay-induced duplicate charge is
// still PENDING at Square must be left alone — re-playing its expired
// idempotency key can land ANOTHER charge, and blind-failing or clawing
// back while the refund is non-terminal would reverse money that may
// still come back. The B1 re-poll pass resolves it on settlement.
if hasInFlightSweepDuplicateRefund(ctx, table, r.ID) {
log.Printf("Stale pending %s row %s has a sweep auto-refund of a duplicate charge still pending at Square — leaving pending until the refund settles", table, r.ID)
continue
}
// B1 (CRITICAL-HIGH): never re-replay a key whose B1 auto-refund of a
// replay-induced duplicate charge has already FAILED. hasFailedSweepDuplicateRefund
// catches a refund Square rejected AFTER the sweep accepted it pending —
// the FAILED-webhook demotion clears the in-flight guard above, so without
// this check the next run would re-replay the expired key and mint ANOTHER
// charge. The b1_attempts cap bounds a refund that keeps failing without
// ever writing a refunds row (REJECTED at call time) or keeps erroring
// (transport) — the current row count is re-read from the DB each run.
// A row here is failed (NOT clawed back — the duplicate money may stand
// at Square) with a CRITICAL notification for manual reconciliation.
if hasFailedSweepDuplicateRefund(ctx, table, r.ID) || r.B1Attempts >= b1DuplicateRefundAttemptCap {
if failStaleRow(ctx, table, r.ID) {
resolved++
}
notifyStaleRowCritical(ctx, r)
log.Printf("Stale pending %s row %s has a FAILED or attempt-capped B1 auto-refund of a replay-induced duplicate charge (b1_attempts=%d) — never re-replaying the expired key; marked failed — MANUAL RECONCILIATION REQUIRED: check Square for the duplicate charge and refund it", table, r.ID, r.B1Attempts)
continue
}
if r.CreatedAt.Before(replayExpired) {
// Key retention window already closed — replaying would misread an
// expired key as "never charged". Blind-fail + WARN exactly as the
// legacy sweep did; a till sale's funded gift card is NOT clawed
// back (the charge outcome is unknown, the money may have landed).
// A5a: this blind-fail can be hiding a real charge (the lost
// response the stored key was meant to reconcile), so the admin
// notification centre must surface it too.
if failStaleRow(ctx, table, r.ID) {
resolved++
unverifiable++
notifyStaleRowCritical(ctx, r)
}
log.Printf("Stale pending %s row %s has a stored idempotency key but is already past Square's key retention window — marked failed without a replay reconcile (may have been charged with a lost response)", table, r.ID)
continue
}
switch res, sqPayID := reconcileStalePaymentByKey(ctx, table, r); res {
case staleReconcileCompleted:
if table == "payments" && r.BookingID == nil {
// Gift-card purchase row (C6): the charge landed at Square but
// the card was never delivered. Completing the row blocks the
// same-key retry that delivers the card — leave it pending.
leaveGiftCardPurchasePending(ctx, r)
continue
}
// F3: same money-safety gate as the by-id rescue above — a charge
// Square reports COMPLETED must never be completed on a booking
// that was cancelled during the pending window.
if table == "payments" {
switch gateStalePaymentRescueOnBooking(ctx, r) {
case staleBookingGateRefused:
resolved++
continue
case staleBookingGateUnknown:
continue
}
}
if rescueKeyedStaleRowCompleted(ctx, table, r, sqPayID) {
resolved++
completed++
}
case staleReconcileLeavePending:
// Ambiguous replay — the charge may still be in flight. Leave the
// row pending; the next sweep run reconciles it again.
log.Printf("Stale pending %s row %s left pending (replay-by-key reconcile ambiguous) — will retry next sweep", table, r.ID)
case staleReconcileDefinitivelyFailed:
// Square proved the charge never happened (key unknown / FAILED) —
// a till sale's funded gift card is clawed back atomically with the
// failed mark, unlike the blind-fail path where the outcome is unknown.
if table == "till_sales" && r.HasGiftCard {
if clawbackTillSaleFunding(ctx, r) {
resolved++
}
continue
}
if failStaleRow(ctx, table, r.ID) {
resolved++
}
case staleReconcileDefinitivelyFailedNoClawback:
// B1 (CRITICAL-HIGH): the auto-refund of the replay-induced
// duplicate charge was definitively REJECTED at Square — the
// duplicate charge stands and the money is REAL at Square, so a
// till sale's funded gift card is NOT clawed back. Mark the parent
// row failed, raise the CRITICAL notification, and set b1_attempts
// to the cap so the expired key is never replayed (each replay
// would mint ANOTHER charge).
if failStaleRow(ctx, table, r.ID) {
resolved++
}
notifyStaleRowCritical(ctx, r)
setB1AttemptsToCap(ctx, table, r.ID)
}
}
return resolved, completed, unverifiable, nil
}
// fetchStaleRows reads the stale pending rows of one table that are older than
// cutoff. keyedOnly restricts the query to rows that can be reconciled by
// replaying a stored idempotency key: those with a key but no square_payment_id
// (rows WITH a square_payment_id are reconciled by payment id in the main
// pass). The legacy till_sales sweep only touches card methods — cash and
// on_the_house are committed synchronously and never pending, but the
// predicate is kept so behaviour is byte-identical for any unexpected row.
func fetchStaleRows(ctx context.Context, table string, cutoff time.Time, keyedOnly bool) ([]staleRow, error) {
keyedPredicate := ""
if keyedOnly {
keyedPredicate = ` AND idempotency_key IS NOT NULL AND square_payment_id IS NULL`
}
methodFilter := ""
if table == "till_sales" {
methodFilter = ` AND payment_method IN ('online_square', 'in_person_card')`
}
var rows pgx.Rows
var err error
if table == "till_sales" {
rows, err = db.Conn.Query(ctx, `
SELECT ts.id, COALESCE(ts.square_payment_id, ''), COALESCE(ts.idempotency_key, ''),
COALESCE(ts.square_source_id, ''), COALESCE(ts.square_request_snapshot, ''),
ts.created_at, ts.item_id, gc.redeemed_by,
(ts.created_at = gc.created_at) AS is_create,
(gc.id IS NOT NULL) AS has_gift_card, ts.total_amount, ts.b1_attempts
FROM till_sales ts
LEFT JOIN gift_cards gc ON gc.id = ts.item_id
WHERE ts.status = 'pending' AND ts.created_at < $1`+methodFilter+keyedPredicate+`
`, cutoff)
} else {
// table is an internal constant ("payments"), never user input, but the
// identifier is routed through pgx.Identifier.Sanitize — the same
// treatment failStaleRow / rescueStaleRowCompleted give the table name —
// so no raw, unquoted table name is ever concatenated into the statement.
rows, err = db.Conn.Query(ctx, `
SELECT id, COALESCE(square_payment_id, ''), COALESCE(idempotency_key, ''),
COALESCE(square_source_id, ''), COALESCE(square_request_snapshot, ''),
created_at, amount, booking_id, created_by, b1_attempts
FROM `+pgx.Identifier{table}.Sanitize()+`
WHERE status = 'pending' AND created_at < $1`+keyedPredicate+`
`, cutoff)
}
if err != nil {
return nil, err
}
defer rows.Close()
var stale []staleRow
for rows.Next() {
r, scanErr := scanStaleRow(table, rows)
if scanErr != nil {
log.Printf("Failed to scan stale pending row from %s: %v", table, scanErr)
continue
}
stale = append(stale, r)
}
return stale, rows.Err()
}
// scanStaleRow scans one row produced by fetchStaleRows into a staleRow,
// deriving the replay amount (in pence) from the stored pound figure — the
// original CreatePayment amount the replay-by-key must repeat.
func scanStaleRow(table string, rows pgx.Rows) (staleRow, error) {
var r staleRow
if table == "till_sales" {
var itemID, redeemedBy, snapshot sql.NullString
var isCreate *bool
var hasGiftCard bool
if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt,
&itemID, &redeemedBy, &isCreate, &hasGiftCard, &r.TotalAmount, &r.B1Attempts); err != nil {
return r, err
}
r.SquareRequestSnapshot = []byte(snapshot.String)
r.ItemID = itemID.String
if redeemedBy.Valid && redeemedBy.String != "" {
r.RedeemToUserID = &redeemedBy.String
}
r.IsCreate = isCreate != nil && *isCreate
r.HasGiftCard = hasGiftCard
r.AmountPence = int64(math.Round(r.TotalAmount * 100))
return r, nil
}
var amount float64
var bookingID, createdBy, snapshot sql.NullString
if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &amount, &bookingID, &createdBy, &r.B1Attempts); err != nil {
return r, err
}
r.SquareRequestSnapshot = []byte(snapshot.String)
r.AmountPence = int64(math.Round(amount * 100))
if bookingID.Valid && bookingID.String != "" {
b := bookingID.String
r.BookingID = &b
}
if createdBy.Valid && createdBy.String != "" {
c := createdBy.String
r.CreatedBy = &c
}
return r, nil
}
// failStaleRow marks one stale pending row 'failed'. Returns true when the row
// was updated (status was still 'pending').
func failStaleRow(ctx context.Context, table, id string) bool {
// table is an internal constant ("payments"/"till_sales"), never user
// input, but the identifier is routed through pgx.Identifier.Sanitize so no
// raw, unquoted table name is ever concatenated into the statement.
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, id)
if err != nil {
log.Printf("Failed to mark stale pending row %s failed: %v", id, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// rescueStaleRowCompleted marks one stale pending row (whose square_payment_id
// already resolves to a COMPLETED charge) 'completed'. For a payments row with
// a booking this mirrors the live-path recording semantics (MEDIUM-3): the
// row's charge is re-split through buildSplitRecords so any overflow beyond
// the booking's remaining obligation is carved out as its own
// payment_type='tip' record (never mis-booked as service revenue), and the
// fully-paid completion check runs so a rescue that settles the booking
// triggers the completion side-effects exactly like the live path. Returns
// true when the row was updated.
func rescueStaleRowCompleted(ctx context.Context, table string, r staleRow) bool {
return rescueStaleRowCompletedTx(ctx, table, r.ID, "", r)
}
// rescueKeyedStaleRowCompleted marks a keyed stale pending row 'completed' with
// the square_payment_id returned by the replay — the lost-response rescue — and
// applies the same split/completion semantics as the by-id rescue above
// (MEDIUM-3). Returns true when the row was updated.
func rescueKeyedStaleRowCompleted(ctx context.Context, table string, r staleRow, squarePaymentID string) bool {
return rescueStaleRowCompletedTx(ctx, table, r.ID, squarePaymentID, r)
}
// rescueStaleRowCompletedTx is the shared implementation behind both rescue
// paths. The split records are computed BEFORE the status flip (while the row
// is still 'pending', so GetBookingPaymentInfo's paid ledger excludes it —
// exactly where the live path computes buildSplitRecords), and the flip +
// split insert + completion run in ONE transaction so a crash can never leave
// the row completed but the carve missing.
func rescueStaleRowCompletedTx(ctx context.Context, table, id, squarePaymentID string, r staleRow) bool {
tx, err := db.Conn.Begin(ctx)
if err != nil {
log.Printf("Failed to begin rescue transaction for stale pending %s row %s: %v", table, id, err)
return false
}
defer func() {
if rErr := tx.Rollback(ctx); rErr != nil && !errors.Is(rErr, pgx.ErrTxClosed) {
log.Printf("Failed to rollback rescue transaction for stale pending %s row %s: %v", table, id, rErr)
}
}()
// Pre-compute the split while the row is still 'pending' so the booking's
// paid ledger excludes it (mirroring the live post-charge path, where
// buildSplitRecords runs before the primary row is marked completed).
records := buildStaleRescueRecords(ctx, tx, r, squarePaymentID)
var tag pgconn.CommandTag
if squarePaymentID != "" {
tag, err = tx.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', square_payment_id = $1, updated_at = NOW()
WHERE id = $2 AND status = 'pending'
`, squarePaymentID, id)
} else {
tag, err = tx.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'completed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, id)
}
if err != nil {
log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err)
return false
}
if int(tag.RowsAffected()) == 0 {
// Already resolved concurrently — nothing to do.
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("Failed to commit rescue no-op for stale pending %s row %s: %v", table, id, cErr)
}
return false
}
if len(records) > 0 {
applyStaleRescueRecords(ctx, tx, r, records)
}
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("Failed to commit rescue of stale pending %s row %s: %v", table, id, cErr)
return false
}
return true
}
// buildStaleRescueRecords computes the payment records a rescued stale pending
// booking payment should be split into (MEDIUM-3). It mirrors the live
// post-charge path: buildSplitRecords partitions the row's charge into the
// deposit/balance/full booking portion plus a carved payment_type='tip' record
// for any overflow beyond the booking's remaining obligation — so a rescue can
// never mis-book the overflow as service revenue. The row MUST still be
// 'pending' when this runs (the paid ledger must exclude it). Rows that need
// no split (tip-type rows, rows without a booking, till_sales) return nil.
// Any error is logged and nil is returned: the row is still rescued (status
// flip) un-split — never worse than the pre-fix minimal flip — and surfaces in
// the log for manual reconciliation.
func buildStaleRescueRecords(ctx context.Context, tx pgx.Tx, r staleRow, squarePaymentID string) []PaymentRecord {
if r.BookingID == nil {
return nil
}
var paymentType, paymentMethod string
var createdBy sql.NullString
if err := tx.QueryRow(ctx, `SELECT payment_type, payment_method, created_by FROM payments WHERE id = $1 FOR UPDATE`, r.ID).Scan(&paymentType, &paymentMethod, &createdBy); err != nil {
log.Printf("MEDIUM-3: failed to read payment row %s for rescue split (booking %s): %v — the row is completed un-split; manual reconciliation recommended", r.ID, *r.BookingID, err)
return nil
}
if paymentType == "tip" {
// A tip row is already tip-type — no carve is needed (a tip can never
// fully pay a booking, so the completion check would be a no-op too).
return nil
}
info, err := NewPaymentService().GetBookingPaymentInfo(ctx, *r.BookingID)
if err != nil || info == nil {
log.Printf("MEDIUM-3: failed to load booking info for rescue split of payment %s (booking %s): %v — the row is completed un-split; manual reconciliation recommended", r.ID, *r.BookingID, err)
return nil
}
amount := float64(r.AmountPence) / 100.0
spID := r.SquarePaymentID
if squarePaymentID != "" {
spID = squarePaymentID
}
var createdByPtr *string
if createdBy.Valid && createdBy.String != "" {
c := createdBy.String
createdByPtr = &c
}
primary := PaymentRecord{
BookingID: *r.BookingID,
PaymentType: paymentType,
PaymentMethod: paymentMethod,
Status: "completed",
Amount: amount,
SquarePaymentID: &spID,
CreatedBy: createdByPtr,
CreatedAt: clock.Now(),
UpdatedAt: clock.Now(),
}
if r.IdempotencyKey != "" {
k := r.IdempotencyKey
primary.IdempotencyKey = &k
}
return buildSplitRecords(primary, paymentType, info, amount)
}
// applyStaleRescueRecords applies the pre-computed split records inside the
// rescue transaction: the primary row is aligned to records[0] and the
// remaining split records (balance/full portion + any carved tip) are inserted
// with their derived idempotency keys. The fully-paid completion check then
// runs so a rescue that settles the booking triggers the completion
// side-effects (loyalty stamps, campaigns, deposits_required) the live path
// would have run. Errors are logged with the row left as-is for manual
// reconciliation — the money was already completed at Square, so the rescue
// must never abort on a bookkeeping failure.
func applyStaleRescueRecords(ctx context.Context, tx pgx.Tx, r staleRow, records []PaymentRecord) {
if len(records) == 0 {
return
}
if _, upErr := tx.Exec(ctx, `
UPDATE payments SET
amount = $1,
payment_type = $2,
fees = $3,
is_vat_applicable = FALSE,
vat_rate = NULL,
vat_amount = NULL,
net_amount = NULL,
updated_at = NOW()
WHERE id = $4
`, records[0].Amount, records[0].PaymentType, records[0].Fees, r.ID); upErr != nil {
log.Printf("MEDIUM-3: failed to align rescued payment %s to its split primary (%v) — manual reconciliation recommended", r.ID, upErr)
return
}
svc := NewPaymentService()
for _, rec := range records[1:] {
if _, cErr := svc.CreatePaymentRecordTx(ctx, tx, rec, nil); cErr != nil {
log.Printf("MEDIUM-3: failed to insert rescue split record for payment %s (%v) — manual reconciliation recommended", r.ID, cErr)
return
}
}
if bookingIsFullyPaid(ctx, tx, *r.BookingID) {
completeActiveBookingFromPayment(ctx, tx, *r.BookingID)
}
}
// staleBookingGate is the outcome of the pre-completion booking-status recheck
// for a stale pending payment row whose charge Square reports COMPLETED.
type staleBookingGate int
const (
// staleBookingGateAllowed — the booking is still payable; the rescue may proceed.
staleBookingGateAllowed staleBookingGate = iota
// staleBookingGateRefused — the booking is cancelled/lapsed/no-show; the row
// was marked FAILED and a critical admin notification raised.
staleBookingGateRefused
// staleBookingGateUnknown — the booking status could not be read; the row was
// left pending (never completed on an unknown state).
staleBookingGateUnknown
)
// gateStalePaymentRescueOnBooking re-reads the booking of a stale pending
// payment row before the sweep rescues it to 'completed' on a Square COMPLETED
// reconcile. It mirrors the live-path guard bookingStatusAllowsCompletedPayment
// (handlers.go): a charge that lands AFTER the booking was cancelled must never
// be silently completed — the cancellation refund path computes refunds from
// completed payments and would miss it, charging a customer for a cancelled
// booking with NO automatic refund (F3).
//
// The booking status is re-read FOR UPDATE inside a transaction so the
// decision serializes against a concurrent cancellation (C5 pattern, mirrors
// recordTerminalPaymentTx). On a cancelled/lapsed/no-show booking the payment
// row is marked FAILED (not completed) and a critical admin notification is
// raised, atomically with the decision, so an operator refunds the customer
// manually. A booking whose status cannot be read is never completed — the row
// is left pending for the next sweep run. Rows with no booking (till_sales;
// gift-card purchases are diverted by the callers before this helper) are
// never gated.
func gateStalePaymentRescueOnBooking(ctx context.Context, r staleRow) staleBookingGate {
if r.BookingID == nil {
return staleBookingGateAllowed
}
tx, err := db.Conn.Begin(ctx)
if err != nil {
log.Printf("CRITICAL: failed to begin booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err)
return staleBookingGateUnknown
}
defer func() {
if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
log.Printf("Failed to rollback booking-status recheck for stale pending payment %s: %v", r.ID, err)
}
}()
var status string
if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, *r.BookingID).Scan(&status); err != nil {
// Booking gone or unreadable: the money state is unknown. Never
// complete on an unknown state — a completed payment on a vanished
// booking would strand the charge outside the refund system.
log.Printf("CRITICAL: Square reports stale pending payment %s COMPLETED but re-reading booking %s status failed (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, err)
return staleBookingGateUnknown
}
if bookingStatusAllowsCompletedPayment(status) {
// Payable — release the booking lock; the caller rescues the row.
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("CRITICAL: failed to commit booking-status recheck for stale pending payment %s (booking %s): %v — leaving pending — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, cErr)
return staleBookingGateUnknown
}
return staleBookingGateAllowed
}
// Cancelled / lapsed / no-show booking: completing the payment would
// charge a customer for a booking the cancellation flow already closed,
// with NO automatic refund. Mark the row FAILED (not completed) and alert
// ops so an operator refunds the customer manually.
tag, upErr := tx.Exec(ctx, `
UPDATE `+pgx.Identifier{"payments"}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, r.ID)
if upErr != nil {
log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — marking the row failed errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, upErr)
return staleBookingGateUnknown
}
if cErr := tx.Commit(ctx); cErr != nil {
log.Printf("CRITICAL: Square payment for stale pending row %s is COMPLETED but booking %s is %q — committing the failed mark errored (%v) — MANUAL RECONCILIATION REQUIRED", r.ID, *r.BookingID, status, cErr)
return staleBookingGateUnknown
}
if int(tag.RowsAffected()) > 0 {
insertCriticalPaymentNotification(ctx, r.BookingID, r.CreatedBy)
log.Printf("CRITICAL: stale pending payment %s is COMPLETED at Square but booking %s is %q — payment marked FAILED instead of completed; operator must refund the customer manually", r.ID, *r.BookingID, status)
return staleBookingGateRefused
}
// The row was already resolved concurrently — nothing left to refuse.
return staleBookingGateUnknown
}
// clawbackTillSaleFunding claws back a stale pending till sale's funded gift
// card (atomically with the failed mark) after a reconcile PROVED the charge
// never completed — the funding has no charge behind it. A sale with no gift
// card is only marked failed. Returns true when the sale was resolved to
// failed; false when it was already resolved by someone else, had no gift card,
// or the clawback failed (CRITICAL logged).
func clawbackTillSaleFunding(ctx context.Context, r staleRow) bool {
action := "topup"
if r.IsCreate {
action = "create"
}
if revErr := revertGiftCardFunding(ctx, action, r.ItemID, r.TotalAmount, r.RedeemToUserID, r.ID); revErr != nil {
if errors.Is(revErr, errTillSaleNotPending) {
log.Printf("Stale pending till sale %s was already resolved (not pending) — skipping funding clawback", r.ID)
return false
}
log.Printf("CRITICAL: failed to claw back gift card %s funding for stale till sale %s: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", r.ItemID, r.ID, revErr)
return false
}
return true
}
// replayRescueLowerBoundSkew is the lower-bound tolerance for a replayed
// COMPLETED payment to still be treated as the ORIGINAL charge under a retained
// key rather than a provably-new duplicate. Rows are inserted pending-first, so
// natural order is row.CreatedAt < Square's created_at by ~0.5-1s — and the DB
// clock can run AHEAD of Square's (independent NTP drift, VM pause/resume),
// which makes a retained-key dedup return the ORIGINAL charge with created <
// r.CreatedAt. Without this tolerance the sweep would declare that original
// payment a "new charge" and auto-refund a charge the customer legitimately
// authorized. A payment created within the skew BEFORE the row is therefore the
// ORIGINAL and is rescued (Loop B MEDIUM — a >1min-ahead DB clock previously
// stranded such a charge pending until the 24h blind-fail); only a payment
// created more than the skew before the row, or after the sweep's replay
// instant (replayAt), is ambiguous / a provable expired-key duplicate.
const replayRescueLowerBoundSkew = 5 * time.Minute
// replayMatchesRowAmount reports whether the replayed payment charged the same
// amount the pending row records — the amount the sweep's replay body repeats
// and the amount any same-key retry MUST reuse (the retry path rejects a
// different amount). A replayed payment carrying a DIFFERENT amount cannot be
// the charge this row is waiting on and must never be rescued onto it. A
// payment with no amount (zero — test fixtures; real Square payments always
// carry one) is not refused here: the lag window below is the primary guard.
func replayMatchesRowAmount(r staleRow, pr *square.PaymentResult) bool {
return pr == nil || pr.Amount == 0 || pr.Amount == r.AmountPence
}
// replayWithinLegitimateWindow reports whether a replayed COMPLETED payment is
// the REAL charge this pending row is waiting on — the ORIGINAL charge under a
// retained key (created ~at row creation) or a later SAME-KEY RETRY charge
// (created between the row's creation and the sweep's replay, F2). The amount
// must match the row (a retry can never change it) and the payment must have
// been created BEFORE the sweep's own replay (replayAt): a payment created at
// or after replayAt can only be the sweep's own expired-key replay-induced
// charge (the sweep just created it), so it is NOT legitimate. The source is
// matched by construction: the replay body is rebuilt from the row's stored
// square_request_snapshot with the LIVE square_source_id override, so a
// payment returned by the replay necessarily charged the row's source (Square's
// PaymentResult does not echo the source id back, so it cannot be compared
// directly).
func replayWithinLegitimateWindow(r staleRow, replayAt time.Time, pr *square.PaymentResult) bool {
if !replayMatchesRowAmount(r, pr) {
return false
}
created, ok := parseReplayedCreatedAt(pr)
if !ok || r.CreatedAt.IsZero() {
return false
}
// Lower bound: replayRescueLowerBoundSkew BEFORE the row — a DB clock ahead
// of Square's can make a retained-key dedup return the ORIGINAL charge with
// created_at slightly before the pending row. Such a payment cannot be a
// provably-new expired-key replay (those land AT the sweep's replay, well
// after the row), so within the skew tolerance it is the legitimate
// original and must be rescued — not left pending to strand the customer's
// authorized charge (Loop B MEDIUM).
if created.Before(r.CreatedAt.Add(-replayRescueLowerBoundSkew)) {
return false
}
// Upper bound: the sweep's OWN replay instant. A payment created strictly
// before the replay is the original or a legit retry; created at/after the
// replay is the sweep's own creation (expired-key duplicate). Comparing
// against replayAt instead of a fixed row-age window makes the decision
// independent of Square's unverified ~24h key-retention window.
return created.Before(replayAt)
}
// isSavedCardSource reports whether a Square source id is a card-on-file
// (saved-card) reference. Only a ccof: source stays valid for recharging long
// after the charge attempt that stored it: a cnon: nonce is single-use, so an
// expired-key replay of a cnon: source is always rejected (proof the charge
// never happened), while a replay of a still-valid ccof: source can land a NEW
// charge under the expired key.
func isSavedCardSource(source string) bool {
return strings.HasPrefix(source, "ccof:")
}
// parseReplayedCreatedAt parses a PaymentResult's ISO 8601 CreatedAt into the
// instant the replayed payment was created at Square. Real Square always
// returns created_at on a payment, so an empty or unparseable value is a
// client/response anomaly.
func parseReplayedCreatedAt(pr *square.PaymentResult) (time.Time, bool) {
if pr == nil || pr.CreatedAt == "" {
return time.Time{}, false
}
created, err := time.Parse(time.RFC3339, pr.CreatedAt)
if err != nil {
return time.Time{}, false
}
return created, true
}
// replayRevealsNewCharge reports whether a COMPLETED keyed replay returned a
// NEW charge rather than the ORIGINAL payment under a retained idempotency key
// (finding A1). A retained-key dedup returns the original payment, created at
// the same instant the pending row was created; a NEW charge made by an
// expired-key replay (Square's ~24h key retention is UNVERIFIED —
// square_http_client.go:626) against the still-valid ccof: source is created
// AT THE SWEEP'S OWN REPLAY MOMENT. A same-key RETRY (handlers.go:1579-1591)
// is a legitimate exception: the retry's charge is created between the row's
// creation and the sweep's replay, so a replayed payment created BEFORE the
// sweep's replay is the REAL charge and must be rescued (F2). Refusal is
// money-safe: a replayed payment that cannot be proven to be the original (or
// a retry created before the sweep's replay) is never rescued (the row stays
// pending, a CRITICAL log is raised and an admin notification inserted), so a
// hidden second charge can never masquerade as the original one. The lower
// bound of the legitimate window is replayRescueLowerBoundSkew BEFORE the row:
// a DB clock ahead of Square's can make the retained-key original look slightly
// older, and such a payment is the legitimate original, not a new charge.
//
// The discriminator is the SWEEP'S OWN REPLAY TIMESTAMP (replayAt), captured
// immediately before SquareClient.ReplayPaymentByKey. This is deliberately NOT
// a fixed row-age window: previous iterations set a constant (22h then 24h,
// matching stalePendingKeyedAge) and each review flipped it because the true
// cutoff is "when did THIS sweep replay the key" — a payment created at/after
// replayAt can only be the sweep's own expired-key creation, and a payment
// created before it is the original or a legit retry, independent of Square's
// actual (unverified) retention window.
//
// The check runs ONLY against real Square timestamps: it is gated off in an
// explicit dev/mock env because the dev mock returns payments whose CreatedAt
// is the mock's "now" at seed/replay time, uncorrelated with the aged
// created_at the test rows carry (the keyed-reconcile tests age rows 23h while
// seeding the Square payment at test time, and a retained-key dedup returns
// that seeded payment). The gate mirrors the snapshot-decryption gate, which
// also runs only in a non-dev/mock env.
func replayRevealsNewCharge(r staleRow, replayAt time.Time, pr *square.PaymentResult) (newCharge bool, created time.Time, createdOK bool) {
if IsExplicitDevOrMockEnv() {
return false, time.Time{}, false
}
created, createdOK = parseReplayedCreatedAt(pr)
if !createdOK || r.CreatedAt.IsZero() {
// Cannot prove the replayed payment is the original charge — refuse to
// rescue rather than hide a possible second charge.
return true, created, createdOK
}
return !replayWithinLegitimateWindow(r, replayAt, pr), created, true
}
// reconcileStalePaymentByKey asks Square for the authoritative status of the
// charge made under a stale pending row's idempotency key and returns the
// tri-state result. The replay sends an IDENTICAL body to the original charge:
// the stored square_request_snapshot (the FULL request — source_id, key,
// amount and every field the original carried), with the source_id overridden
// by the row's CURRENT square_source_id column value (authoritative — a
// same-key reuse refreshes the column while the snapshot JSON is stale).
// Replaying a partial body would
// return IDEMPOTENCY_KEY_REUSED for a RETAINED key and strand the row pending
// forever. Square's idempotency guarantee returns the ORIGINAL payment for a
// retained key (never a second charge); a COMPLETED payment rescues the row to
// 'completed' with its real payment id. A definitive 4xx rejection
// (ErrReplayKeyNotRetained — Square attempted a real charge with the
// expired/used source and refused it, so the charge never happened) or a
// FAILED/CANCELED payment proves the charge never completed and fails the row.
// IDEMPOTENCY_KEY_REUSED is NEVER proof of no charge: with an identical body it
// can only mean the stored source differs from the original (a data bug), so
// the row is left pending with a CRITICAL log. Any OTHER error (transport /
// 5xx / ambiguous) leaves the row pending — the charge may still have completed
// at Square. The second return value is the Square payment id of the completed
// payment ("" otherwise), written back on a rescue.
//
// ENV CONTRACT (finding 4): the sweep and the charge process MUST run with the
// SAME SQUARE_ENVIRONMENT and SQUARE_LOCATION_ID. The stored snapshot embeds
// the location_id used at charge time; if the env/location changes between a
// charge and its replay, the replayed wire body differs and a RETAINED key
// returns IDEMPOTENCY_KEY_REUSED — stranding every retained-key row pending
// (safe) until the 24h blind-fail. This is a single-location deployment; the
// contract is enforced by ops (same env for the sweeper and the API). The
// environment and location are read through square.SquareEnvironment() and
// square.SquareLocationID() — the SAME code path the charge-time HTTP client
// uses (newHTTPClient resolves its base URL and location from those helpers) —
// so the sweep can never drift to a second, independent env read; the
// reconcile below logs the resolved values at each replay as a tripwire.
func reconcileStalePaymentByKey(ctx context.Context, table string, r staleRow) (staleReconcileResult, string) {
snapshot := r.SquareRequestSnapshot
// fallbackBody is true when the row has NO stored square_request_snapshot
// and the minimal body (key + source + amount) is rebuilt. The rebuilt body
// omits reference_id / customer_id / note / buyer_email_address that the
// original charge carried, so for a RETAINED key real Square compares the
// WHOLE body, sees the difference and returns IDEMPOTENCY_KEY_REUSED
// (stranding the row pending) while the dev mock — which compares only the
// source — returns the original payment (rescuing it). Both are money-safe;
// the divergence is dev-vs-prod OBSERVABILITY only, and pre-launch there
// are no prod legacy rows (every charge now stores its snapshot), so the
// fallback semantics are deliberately left unchanged.
fallbackBody := false
if len(bytes.TrimSpace(snapshot)) == 0 {
fallbackBody = true
// Legacy row without a stored request snapshot — rebuild the minimal
// identical body (key + source + amount) exactly as the pre-snapshot
// replay did. Such rows can still be reconciled as long as the stored
// source/amount match the original charge.
fallback, mErr := json.Marshal(square.CreatePaymentReq{
Amount: r.AmountPence,
Currency: "GBP",
SourceID: r.SquareSourceID,
IdempotencyKey: r.IdempotencyKey,
})
if mErr != nil {
log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body for row %s (%v) — leaving pending", table, r.ID, mErr)
return staleReconcileLeavePending, ""
}
snapshot = fallback
}
// PROD-mode snapshot decryption: since the AES-GCM snapshot work, the stored
// square_request_snapshot is ENCRYPTED in a non-mock deployment
// (SQUARE_ENVIRONMENT production/sandbox — the same gate the 2FA
// enforcement uses, IsExplicitDevOrMockEnv); the dev mock stores plaintext.
// Never replay a corrupt snapshot: a decryption failure is a serious error
// that leaves the row pending with a CRITICAL log for manual reconciliation
// instead of replaying garbage (which could return a misleading answer).
if !fallbackBody && !IsExplicitDevOrMockEnv() {
dec, err := decryptSnapshot(snapshot)
if err != nil {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: failed to decrypt the stored request snapshot for row %s (%v) — leaving pending — MANUAL RECONCILIATION REQUIRED", table, r.ID, err)
}
snapshot = dec
}
// The stored snapshot embeds the source_id of the ORIGINAL charge, but a
// pending row REUSED by a same-key retry has its square_source_id column
// refreshed to the retry's source while the snapshot JSON stays stale (the
// write side now refreshes the snapshot too). Replaying the stale embedded
// source under a key Square already retains for the new one would return
// IDEMPOTENCY_KEY_REUSED and strand the row pending — so rebuild the replay
// body with the LIVE column value, which is authoritative (set at charge
// time, refreshed on every reuse). The minimal snapshot-less fallback body
// above is already built from the live source and needs no override. An
// unparseable snapshot must never look like proof of no charge — leave the
// row pending for manual reconciliation.
if !fallbackBody && r.SquareSourceID != "" {
var req square.CreatePaymentReq
if err := json.Unmarshal(snapshot, &req); err != nil {
log.Printf("Stale pending %s reconcile by key: failed to parse stored request snapshot for row %s to override the source_id (%v) — leaving pending", table, r.ID, err)
return staleReconcileLeavePending, ""
}
req.SourceID = r.SquareSourceID
overridden, mErr := json.Marshal(req)
if mErr != nil {
log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body with the current square_source_id for row %s (%v) — leaving pending", table, r.ID, mErr)
return staleReconcileLeavePending, ""
}
snapshot = overridden
}
// The replay repeats the stored request snapshot — with the live source_id
// override above — so Square's idempotency dedup returns the original
// payment for a retained key. The
// sweep resolves its Square environment and location through the same
// helpers the charge-time HTTP client uses (square.SquareEnvironment /
// square.SquareLocationID), so the replay and the charge can never read
// two different env sources (see the ENV CONTRACT above). The values are
// logged on each replay as a tripwire for env/location drift between the
// sweeper and the API.
log.Printf("[SWEEP] replay-by-key reconcile for %s row %s: SQUARE_ENVIRONMENT=%q SQUARE_LOCATION_ID=%q (must match the charge-time env/location for identical-body idempotency)", table, r.ID, square.SquareEnvironment(), square.SquareLocationID())
// The discriminator for "original/retry vs expired-key duplicate" is the
// sweep's OWN replay instant: any payment Square returns with created_at
// at/after this moment was created by THIS replay call (the key expired and
// Square charged the still-valid source), so it must never be rescued. A
// payment created before this instant is the original or a legit same-key
// retry. Capturing replayAt here — not a fixed row-age window — keeps the
// decision correct regardless of Square's unverified ~24h retention.
replayAt := clock.Now()
pr, err := SquareClient.ReplayPaymentByKey(ctx, snapshot)
if err != nil {
if errors.Is(err, square.ErrReplayKeyNotRetained) {
// A1: a rejection of a replayed charge against a SAVED-CARD (ccof:)
// source is NOT proof the original charge never happened — the same
// key-retention race that lets the replay land a NEW charge can
// leave that charge even when the probe returns a rejection. Blindly
// failing (and clawing back a till sale's funded gift card) on a
// ccof source could reverse money a replay-created charge already
// took. Leave the row pending and alert ops. A cnon-source
// (spent-nonce) rejection REMAINS definitive proof of no charge (a
// single-use nonce cannot be recharged), keeping the proven-failed
// path and its clawback.
if isSavedCardSource(r.SquareSourceID) {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: Square rejected the identical-body replay (no payment under the stored key) but the row's source is a still-valid saved card (ccof:) — the replay may have landed a NEW charge under an expired idempotency key — leaving row %s PENDING without failing/clawing back — MANUAL RECONCILIATION REQUIRED: verify at Square whether a charge exists before re-issuing", table, r.ID)
}
log.Printf("Stale pending %s reconcile by key: Square has no payment under the stored idempotency key (identical-body replay rejected) — marking failed; the charge provably never happened", table)
return staleReconcileDefinitivelyFailed, ""
}
if square.ErrorCode(err) == "IDEMPOTENCY_KEY_REUSED" {
if fallbackBody {
// Snapshot-less fallback row: the minimal rebuilt body cannot be
// identical to the original charge, so real Square rejects the
// retained-key replay while the dev mock (source-aware only)
// rescues it. The stranded row would otherwise be invisible
// until the 24h blind-fail — surface it now (finding 2).
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED via the minimal snapshot-less fallback body (key=%s) — real Square compares the WHOLE request (reference_id/customer_id/note/buyer_email_address absent from the rebuilt body) and rejects the identical-key replay, stranding the row pending; the dev mock would rescue it — dev-vs-prod observability divergence, NOT proof the charge never happened — MANUAL RECONCILIATION REQUIRED", table, r.IdempotencyKey)
}
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED — the stored square_source_id differs from the original charge's source (data bug); this is NOT proof the charge never happened — leaving pending — MANUAL RECONCILIATION REQUIRED", table)
}
log.Printf("Stale pending %s reconcile by idempotency key hit an ambiguous error (%v) — leaving pending for a later sweep run", table, err)
return staleReconcileLeavePending, ""
}
// Square's terminal payment states are COMPLETED, CANCELED, FAILED;
// APPROVED (authorization-only, delayed capture) and PENDING are
// NON-terminal — both can still transition to COMPLETED, so clawing back
// the funded gift card on either would risk reversing a charge that later
// lands. This app always creates payments with autocomplete (default true),
// so it never produces APPROVED/PENDING rows today, but the classification
// must match Square's documented state machine.
switch pr.Status {
case "COMPLETED":
// A1: a replayed COMPLETED payment created long AFTER the pending row
// is a NEW charge Square made against the still-valid source with an
// expired idempotency key (Square's ~24h key retention is UNVERIFIED —
// square_http_client.go:626), NOT the original charge a retained key
// returns. Rescuing the row with the new payment id would hide the
// second charge behind the original. Leave the row pending and alert
// ops so both charges can be reconciled at Square and the duplicate
// refunded.
if newCharge, created, createdOK := replayRevealsNewCharge(r, replayAt, pr); newCharge {
// B1 caveat: auto-refund ONLY a PROVABLY-created-later duplicate.
// An UNPARSEABLE replayed CreatedAt does not prove the payment is a
// duplicate — it could be the ORIGINAL charge a retained key
// returned, whose created_at was lost/corrupt. Auto-refunding that
// would wrongly reverse a legitimately authorized charge, so the
// pre-B1 outcome is restored for this case: leave the row PENDING
// with a CRITICAL notification for a human to reconcile at Square
// (the refund is withheld until a human confirms the second charge
// exists — never auto-issued against a possibly-original payment).
if !createdOK {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s has an UNPARSEABLE created_at — cannot prove it is a NEW expired-key replay rather than the ORIGINAL charge under a retained key — leaving row %s PENDING — MANUAL RECONCILIATION REQUIRED: verify at Square whether this is a second charge before refunding", table, pr.ID, r.ID)
}
// B1 (MEDIUM 2): a replayed COMPLETED payment created BEFORE the
// pending row is NOT provably a new expired-key replay — those land
// ~22h AFTER the row. Only genuinely-before payments reach this
// line: a payment within replayRescueLowerBoundSkew of the row is
// already treated as the ORIGINAL retained-key charge by
// replayWithinLegitimateWindow and rescued. This one is far enough
// before the row that no clock skew can explain it — the row is
// left PENDING with a CRITICAL notification for manual
// reconciliation, never auto-refunded.
if created.Before(r.CreatedAt) {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s was created %s before the pending row %s (beyond the %s clock-skew tolerance) — cannot prove it is a NEW expired-key replay nor the ORIGINAL charge — leaving row %s PENDING — MANUAL RECONCILIATION REQUIRED: verify at Square whether this is a second charge before refunding", table, pr.ID, created.Sub(r.CreatedAt).Round(time.Minute), r.ID, replayRescueLowerBoundSkew, r.ID)
}
lag := created.Sub(r.CreatedAt).Round(time.Minute).String()
// B1: the replayed COMPLETED payment is a REAL charge the customer
// never authorized (an expired-key replay landed it on the still
// valid saved-card source, provably created after the pending row).
// Auto-refund C2 at Square instead of leaving the row pending for a
// human: the amount is r.AmountPence
// (the replay repeats the row's amount), the key is fresh, and the
// reason marks the sweep. When the refund lands, the row is marked
// failed (the original charge was never found — the caller's
// definitively-failed branch does that; a till sale's gift-card
// funding is clawed back there too, since the duplicate has been
// refunded). A refund Square leaves PENDING is NON-terminal: the row
// stays pending (no fail, no clawback) and the B1 re-poll pass
// resolves it when Square settles. Only a refund FAILURE keeps the
// CRITICAL manual-reconciliation path.
if refundErr := refundSweepDuplicateCharge(ctx, table, r, pr); refundErr == nil {
log.Printf("stale pending %s row %s: the replayed COMPLETED payment %s was a NEW charge under an expired idempotency key (lag %s) — auto-refunded the duplicate at Square and marking the row failed (the original charge was never found)", table, r.ID, pr.ID, lag)
return staleReconcileDefinitivelyFailed, ""
} else if errors.Is(refundErr, errSweepRefundPending) {
log.Printf("stale pending %s row %s: the auto-refund of the replay-induced duplicate charge %s is PENDING at Square (non-terminal) — leaving the row pending; the B1 refund re-poll resolves it when Square settles", table, r.ID, pr.ID)
return staleReconcileLeavePending, ""
} else if errors.Is(refundErr, errSweepRefundRejected) {
// B1 (CRITICAL-HIGH): the auto-refund was DEFINITIVELY REJECTED
// at Square — the duplicate charge stands and can never be
// auto-refunded. The parent row is failed WITHOUT a gift-card
// clawback (the money is real at Square), the CRITICAL
// notification is raised, and b1_attempts is set to the cap so
// the expired key is never replayed (each replay would mint
// ANOTHER charge). The outer loop's
// staleReconcileDefinitivelyFailedNoClawback branch does all of
// this.
log.Printf("stale pending %s row %s: the auto-refund of the replay-induced duplicate charge %s was DEFINITIVELY REJECTED at Square (%v) — marking the row failed with a CRITICAL notification; the expired key will never be replayed", table, r.ID, pr.ID, refundErr)
return staleReconcileDefinitivelyFailedNoClawback, ""
} else {
return leavePendingCritical(ctx, r, "stale pending %s reconcile by key: the replayed COMPLETED payment %s was created after the pending row %s (lag %s) — a NEW charge under an expired idempotency key; auto-refund FAILED (%v) — leaving the row PENDING — MANUAL RECONCILIATION REQUIRED: check Square for both charges and refund the duplicate", table, pr.ID, r.ID, lag, refundErr)
}
}
return staleReconcileCompleted, pr.ID
case "CANCELED", "FAILED":
log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed, ""
case "APPROVED", "PENDING":
log.Printf("Stale pending %s is %q at Square (replay by key, non-terminal) — leaving pending for a later sweep run", table, pr.Status)
return staleReconcileLeavePending, ""
default:
log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed, ""
}
}
// errSweepRefundPending is the sentinel refundSweepDuplicateCharge returns when
// Square accepted the auto-refund of a replay-induced duplicate charge but left
// it PENDING — a NON-terminal state: the refund may still complete (the
// duplicate is reversed) or fail. The caller must NOT treat it as success: the
// parent row is left PENDING (never marked failed, a till sale's funded gift
// card is never clawed back) so a later sweep run can re-poll the refund. The
// refund's idempotency key ("sweepdup-"+paymentID) is deterministic, so the
// re-poll finds the SAME refund at Square.
var errSweepRefundPending = errors.New("sweep duplicate-charge refund pending at Square")
// errSweepRefundRejected is the sentinel refundSweepDuplicateCharge returns
// when Square DEFINITIVELY rejected the auto-refund of a replay-induced
// duplicate charge (refund status FAILED/REJECTED). The duplicate charge
// stands at Square and can never be auto-refunded, so the caller must mark the
// parent row failed, raise the CRITICAL notification and set b1_attempts to
// the cap — replaying the expired key would mint ANOTHER charge (CRITICAL-HIGH
// B1). Unlike errSweepRefundPending (non-terminal, row stays pending), this is
// terminal and MUST NOT be left for the next sweep run.
var errSweepRefundRejected = errors.New("sweep duplicate-charge refund rejected at Square")
// sweepDuplicateRefundReason is the audit-trail reason carried by every
// refunds row for a sweep auto-refund of a replay-induced duplicate charge
// (B1). The B1 re-poll pass (sweepPendingB1Refunds, refunds.go) and the
// in-flight guard (hasInFlightSweepDuplicateRefund) match on it, so it must
// stay in lockstep with refunds.go. For a till_sale the reason carries the
// parent till_sale id (see sweepDuplicateRefundReasonFor) — refunds has no
// till_sale column and the refund must link back to the sale the re-poll pass
// claws back when Square settles.
const sweepDuplicateRefundReason = "duplicate charge — sweep replay"
// sweepDuplicateRefundReasonFor returns the refunds-row reason for a sweep
// auto-refund. tillSaleID is "" for payments-table rows (whose parent is the
// refund's own payment_id); a till_sale's parent is encoded in the reason
// because refunds.payment_id is FK'd to payments and the sale has no payments
// row of its own.
func sweepDuplicateRefundReasonFor(tillSaleID string) string {
if tillSaleID == "" {
return sweepDuplicateRefundReason
}
return sweepDuplicateRefundReason + " (till_sale " + tillSaleID + ")"
}
// refundSweepDuplicateCharge auto-refunds a replayed COMPLETED payment that the
// sweep proved to be a NEW charge under an expired idempotency key (B1) — money
// the customer never authorized. The refund reuses the existing Square refund
// path (SquareClient.RefundPayment) with:
//
// - amount pr.Amount — the replayed payment's ACTUAL charged amount. The row's
// stored amount (r.AmountPence) can differ when the original charge applied
// a deposit-with-discount (chargeAmount = requested - discount): the row
// records the discounted charge, and the replay repeats that discounted
// body, so the duplicate charged exactly pr.Amount;
// - a deterministic idempotency key derived from the replayed payment's id,
// so a re-run refunding the SAME duplicate dedups at Square instead of
// issuing a second refund, while a different duplicate on a later replay
// gets a fresh key and is refunded too;
// - reason "duplicate charge — sweep replay" for the audit trail.
//
// A refunds row is recorded for BOTH payments-table and till_sale rows (a
// till_sale's refund attaches to a synthetic payments row created for the
// duplicate charge, since refunds.payment_id is FK'd to payments; the insert
// is idempotent on the deterministic refund key, whose UNIQUE constraint
// doubles as the dedup guard), and an admin notification is inserted so an
// operator sees the auto-refund.
//
// Return semantics:
//
// - nil — the refund COMPLETED at Square; the caller marks the row
// definitively failed (the ORIGINAL charge was never found) and, for a till
// sale, claws back the funded gift card — the duplicate has been reversed;
// - errSweepRefundPending — the refund is PENDING (non-terminal); the caller
// leaves the row pending; the B1 re-poll pass (sweepPendingB1Refunds,
// refunds.go) resolves the row when Square settles;
// - any other error — the money state at Square is untouched; the caller
// keeps the CRITICAL manual-reconciliation path.
func refundSweepDuplicateCharge(ctx context.Context, table string, r staleRow, pr *square.PaymentResult) error {
if pr == nil || pr.ID == "" {
return errors.New("replayed payment has no Square payment id to refund")
}
if pr.Amount <= 0 {
return fmt.Errorf("refusing to auto-refund a non-positive amount %d pence for replayed payment %s (row %s)", pr.Amount, pr.ID, r.ID)
}
// B1: count THIS attempt against the row's b1_attempts cap so a refund
// that keeps failing (REJECTED at call time writes no refunds row; a
// transport error writes none either) can never be replayed forever.
// Each replay mints another charge at Square.
incrementB1Attempts(ctx, table, r.ID)
// Deterministic per-duplicate key: Square dedups same-key refunds, so a
// re-run replaying the same C2 never double-refunds. "sweepdup-" + Square
// payment id stays well under Square's 45-char idempotency-key limit.
refundKey := "sweepdup-" + pr.ID
if len(refundKey) > maxIdempotencyKeyLength {
refundKey = truncateIdempotencyKey("sweepdup", pr.ID)
}
res, refundErr := SquareClient.RefundPayment(ctx, square.RefundPaymentReq{
PaymentID: pr.ID,
Amount: pr.Amount,
IdempotencyKey: refundKey,
Reason: sweepDuplicateRefundReason,
})
if refundErr != nil {
return fmt.Errorf("auto-refund of replay-induced duplicate charge %s failed: %w", pr.ID, refundErr)
}
status := "completed"
pending := false
switch res.Status {
case "PENDING", "APPROVED":
// Square's non-terminal refund states — the money has not moved yet but
// the refund is in flight. NON-terminal: the caller must leave the row
// pending and never claw back a funded gift card (reversing the funding
// before Square settles the refund could leave a customer charged with
// no gift card if the refund later fails).
status = "pending"
pending = true
case "FAILED", "REJECTED":
// Square definitively rejected the refund — the duplicate charge stands
// and can never be auto-refunded. The caller must NOT leave the row
// pending for another replay (a replay mints ANOTHER charge): it marks
// the parent failed, raises the CRITICAL notification and sets the
// b1_attempts cap so the expired key is never replayed.
return fmt.Errorf("auto-refund of replay-induced duplicate charge %s was %s at Square: %w", pr.ID, res.Status, errSweepRefundRejected)
}
recordSweepDuplicateRefundRow(ctx, table, r, pr, res.ID, status, refundKey)
insertCriticalPaymentNotification(ctx, r.BookingID, r.CreatedBy)
if pending {
log.Printf("Auto-refund of replay-induced duplicate charge %s (%d pence) for pending row %s is PENDING at Square (refund %s) — leaving the row pending for the refund re-poll", pr.ID, pr.Amount, r.ID, res.ID)
return errSweepRefundPending
}
log.Printf("Auto-refunded replay-induced duplicate charge %s (%d pence) for pending row %s — refund %s", pr.ID, pr.Amount, r.ID, res.ID)
return nil
}
// recordSweepDuplicateRefundRow writes the refunds row for a sweep auto-refund
// of a replay-induced duplicate charge (B1), idempotently on the deterministic
// refund idempotency key (UNIQUE — a re-run can never mint a second row). For
// a payments-table row the refund attaches to the pending payment row itself.
// A till_sale has no payments row (refunds.payment_id is NOT NULL + FK), so a
// synthetic payments row is created FIRST — 'completed' (the duplicate charge
// genuinely landed at Square) with the duplicate's square_payment_id, so the
// B1 re-poll pass can look the refund up by payment id — and the refund is
// attached to it. The parent till_sale id is carried in the reason so the
// re-poll pass claws it back when Square settles.
func recordSweepDuplicateRefundRow(ctx context.Context, table string, r staleRow, pr *square.PaymentResult, refundID, status, refundKey string) {
amountPounds := float64(pr.Amount) / 100.0
paymentID := r.ID
var bookingID *string
reason := sweepDuplicateRefundReason
if table == "payments" {
bookingID = r.BookingID
} else {
// till_sale: create the synthetic payments row for the duplicate charge.
payKey := truncateIdempotencyKey("sweepdup-pay", pr.ID)
err := db.Conn.QueryRow(ctx, `
INSERT INTO payments (payment_type, payment_method, status, amount, square_payment_id, idempotency_key, created_by, created_at)
VALUES ('full', 'in_person_card', 'completed', $1, $2, $3, $4, NOW())
ON CONFLICT (idempotency_key) DO NOTHING
RETURNING id
`, amountPounds, pr.ID, payKey, r.CreatedBy).Scan(&paymentID)
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but creating the payments row for till sale %s failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, r.ID, err)
return
}
// Re-run: the synthetic row already exists — reuse it.
if rErr := db.Conn.QueryRow(ctx, `SELECT id FROM payments WHERE idempotency_key = $1`, payKey).Scan(&paymentID); rErr != nil {
log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but re-reading the payments row for till sale %s failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, r.ID, rErr)
return
}
}
reason = sweepDuplicateRefundReasonFor(r.ID)
}
if _, insErr := db.Conn.Exec(ctx, `
INSERT INTO refunds (payment_id, booking_id, amount, square_refund_id, status, origin, reason, idempotency_key, created_by, created_at)
VALUES ($1, $2, $3, $4, $5, 'manual', $6, $7, $8, NOW())
ON CONFLICT (idempotency_key) DO UPDATE SET status = EXCLUDED.status, square_refund_id = EXCLUDED.square_refund_id
`, paymentID, bookingID, amountPounds, refundID, status, reason, refundKey, r.CreatedBy); insErr != nil {
log.Printf("CRITICAL: auto-refunded duplicate charge %s at Square (refund %s) but recording the refunds row failed: %v — MANUAL RECONCILIATION REQUIRED", pr.ID, refundID, insErr)
}
}
// hasInFlightSweepDuplicateRefund reports whether a stale pending row carries a
// sweep auto-refund of a replay-induced duplicate charge (B1) that Square left
// PENDING. While the refund is in flight the row must NOT be replayed (a replay
// of the expired-key row could land ANOTHER charge), blind-failed or clawed
// back (the refund is non-terminal — money may still reverse). The B1 re-poll
// pass (sweepPendingB1Refunds, refunds.go) resolves the parent row when Square
// settles. For a till_sale the refund row's reason carries the sale id (see
// sweepDuplicateRefundReasonFor); a payments-table refund's payment_id IS the
// parent row.
func hasInFlightSweepDuplicateRefund(ctx context.Context, table, id string) bool {
var exists bool
var err error
if table == "till_sales" {
err = db.Conn.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM refunds
WHERE status = 'pending' AND square_refund_id IS NOT NULL
AND reason = $1
)
`, sweepDuplicateRefundReasonFor(id)).Scan(&exists)
} else {
err = db.Conn.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM refunds
WHERE payment_id = $1 AND status = 'pending' AND square_refund_id IS NOT NULL
AND reason = $2
)
`, id, sweepDuplicateRefundReason).Scan(&exists)
}
if err != nil {
log.Printf("Failed to check for an in-flight sweep duplicate refund on %s row %s: %v", table, id, err)
return false
}
return exists
}
// hasFailedSweepDuplicateRefund reports whether a stale pending row carries a
// sweep auto-refund of a replay-induced duplicate charge (B1) that Square
// definitively REJECTED (refunds row status 'failed'). The FAILED-webhook
// demotion (refund.updated → status failed) or the B1 re-poll's terminal-failed
// resolution marks a refund that was accepted PENDING and later failed — which
// clears the in-flight guard (hasInFlightSweepDuplicateRefund only matches
// 'pending'). Replaying such a row's expired key would mint ANOTHER charge, so
// the sweep must never do it (CRITICAL-HIGH B1). A till_sale's reason carries
// the sale id (see sweepDuplicateRefundReasonFor), mirroring the in-flight
// guard's scoping.
func hasFailedSweepDuplicateRefund(ctx context.Context, table, id string) bool {
var exists bool
var err error
if table == "till_sales" {
err = db.Conn.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM refunds
WHERE status = 'failed' AND square_refund_id IS NOT NULL
AND reason = $1
)
`, sweepDuplicateRefundReasonFor(id)).Scan(&exists)
} else {
err = db.Conn.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM refunds
WHERE payment_id = $1 AND status = 'failed' AND square_refund_id IS NOT NULL
AND reason = $2
)
`, id, sweepDuplicateRefundReason).Scan(&exists)
}
if err != nil {
log.Printf("Failed to check for a failed sweep duplicate refund on %s row %s: %v", table, id, err)
return false
}
return exists
}
// incrementB1Attempts records one more B1 auto-refund attempt against a stale
// pending row's b1_attempts cap. Runs BEFORE the Square refund call so every
// attempt (success, pending, rejected, transport error) counts — a refund that
// keeps failing without writing a refunds row can never exceed the cap and
// force an unbounded replay loop. Best-effort: a failed UPDATE must not block
// the refund itself.
func incrementB1Attempts(ctx context.Context, table, id string) {
if _, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET b1_attempts = b1_attempts + 1, updated_at = NOW()
WHERE id = $1
`, id); err != nil {
log.Printf("Failed to increment b1_attempts on %s row %s: %v", table, id, err)
}
}
// setB1AttemptsToCap pins a stale pending row's b1_attempts to the cap after a
// B1 auto-refund was definitively REJECTED at Square — the belt-and-suspenders
// guarantee that the expired key is never replayed even if the row somehow
// stays pending. Best-effort like incrementB1Attempts.
func setB1AttemptsToCap(ctx context.Context, table, id string) {
if _, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET b1_attempts = $2, updated_at = NOW()
WHERE id = $1 AND b1_attempts < $2
`, id, b1DuplicateRefundAttemptCap); err != nil {
log.Printf("Failed to set b1_attempts cap on %s row %s: %v", table, id, err)
}
}
// leaveGiftCardPurchasePending keeps a gift-card-purchase payment row (payments
// table, booking_id NULL) pending after Square confirms the charge COMPLETED,
// instead of rescuing it to 'completed'. Completing the row would permanently
// block the same-key retry (BuyGiftCard) that reuses the pending record to
// deliver the card — the customer would stay charged with no gift card (C6).
// The row is left pending with a CRITICAL log + admin notification so the retry
// can still deliver the card and an admin is alerted to reconcile manually.
func leaveGiftCardPurchasePending(ctx context.Context, r staleRow) {
log.Printf("CRITICAL: gift card purchase payment %s is COMPLETED at Square but the gift card was never issued (issue transaction failed, retry abandoned) — leaving the payment PENDING so a same-key retry can still deliver the card — MANUAL RECONCILIATION REQUIRED", r.ID)
insertCriticalPaymentNotification(ctx, nil, r.CreatedBy)
}
// insertCriticalPaymentNotification surfaces an unresolved money event in the
// admin notification centre (reason 'critical_payment_log' — the DB-backed
// stand-in for the un-watched CRITICAL payment logs). bookingID is set when the
// issue ties to a booking (untracked terminal charges); userID is set when it
// ties to a user (gift-card purchases). The NOT EXISTS guard keeps ONE
// notification per issue instead of one per sweep run, and requires the prior
// notification to be unacknowledged (acknowledged_at IS NULL) so that after an
// admin acknowledges it, a NEW event for the same booking/user re-notifies.
//
// This is the SWEEP-specific variant of a same-named helper in the webhooks
// package (handlers/webhooks/square.go, insertCriticalPaymentNotification) with
// an INTENTIONAL scoping difference: this one dedups on (reason, booking_id,
// user_id) — booking and/or user — because sweep-originated events may be
// user-attributed without a booking (gift-card purchases) or booking-attributed
// without a user (untracked terminal charges). The webhook variant takes
// (ctx, bookingID, disputeID string), never writes user_id, and dedups on
// (reason, booking_id) or a deterministic per-dispute id. Both share the
// admin_notifications table and the 'critical_payment_log' reason but serve
// different call paths (sweep vs webhook) — do not merge them, and if the
// NOT EXISTS guard shape ever changes, update BOTH copies.
func insertCriticalPaymentNotification(ctx context.Context, bookingID, userID *string) {
tag, err := db.Conn.Exec(ctx, `
INSERT INTO admin_notifications (reason, booking_id, user_id, created_at)
SELECT 'critical_payment_log', $1, $2, NOW()
WHERE NOT EXISTS (
SELECT 1 FROM admin_notifications an
WHERE an.reason = 'critical_payment_log'
AND an.booking_id IS NOT DISTINCT FROM $1
AND an.user_id IS NOT DISTINCT FROM $2
AND an.acknowledged_at IS NULL
)
`, bookingID, userID)
if err != nil {
log.Printf("Failed to insert admin_notifications for critical payment issue (booking=%v user=%v): %v", bookingID, userID, err)
return
}
if int(tag.RowsAffected()) > 0 {
log.Printf("Inserted critical-payment admin notification (booking=%v user=%v)", bookingID, userID)
}
}
// notifyStaleRowCritical inserts a critical-payment admin notification for a
// stale row via the shared insertCriticalPaymentNotification helper — the
// notification's deterministic dedup key is the user attribution (the NOT
// EXISTS guard keeps ONE notification per user instead of one per sweep run).
// Payments rows are attributed by the payer (created_by); till_sales rows by
// the funded gift card's redeemed-to user when there is one. booking_id is
// deliberately not used: the sweep runs after the charge process and a
// booking-attributed notification would pin the booking row through the
// admin_notifications FK (no cascade) for as long as the notification survives.
func notifyStaleRowCritical(ctx context.Context, r staleRow) {
var userID *string
if r.CreatedBy != nil {
userID = r.CreatedBy
} else {
userID = r.RedeemToUserID
}
insertCriticalPaymentNotification(ctx, nil, userID)
}
// leavePendingCritical is the shared terminal outcome for a stale pending row
// that cannot be safely resolved by the sweep: it logs a CRITICAL line (the
// caller's existing message), raises the deduped critical-payment admin
// notification via notifyStaleRowCritical, and returns
// staleReconcileLeavePending so the row stays pending for a human. Every
// "do not touch, an operator must reconcile" branch — snapshot-decrypt
// failure, the ccof replay rejection, IDEMPOTENCY_KEY_REUSED and the
// replayed-new-charge case (A4) — now runs the identical log + notification
// outcome through this one helper.
func leavePendingCritical(ctx context.Context, r staleRow, format string, args ...any) (staleReconcileResult, string) {
notifyStaleRowCritical(ctx, r)
log.Printf("CRITICAL: "+format, args...)
return staleReconcileLeavePending, ""
}
// staleReconcileResult is the tri-state outcome of reconciling one stale
// pending row against Square. Only a definitively-resolved outcome touches the
// row: an ambiguous answer (transport error / 5xx) leaves it pending so a
// same-key retry can still reuse it if the charge actually completed.
type staleReconcileResult int
const (
// staleReconcileLeavePending — Square's answer was ambiguous; the row stays
// pending for the next sweep run.
staleReconcileLeavePending staleReconcileResult = iota
// staleReconcileCompleted — Square confirms the charge completed; rescue
// the row to 'completed'.
staleReconcileCompleted
// staleReconcileDefinitivelyFailed — Square proves the charge never
// completed (payment not found / non-completed status); mark the row
// 'failed' exactly as the legacy bulk sweep did.
staleReconcileDefinitivelyFailed
// staleReconcileDefinitivelyFailedNoClawback — Square definitively rejected
// the B1 auto-refund of a replay-induced duplicate charge (the duplicate
// stands at Square, so the money is REAL and a till sale's funded gift card
// must NOT be clawed back). The outer loop marks the parent failed without a
// clawback, raises the CRITICAL notification and sets the b1_attempts cap.
staleReconcileDefinitivelyFailedNoClawback
)
// reconcileStalePaymentAtSquare asks Square for the authoritative status of a
// stale pending charge and returns the tri-state result. A COMPLETED payment
// rescues the row to 'completed'; a NOT_FOUND error or any non-COMPLETED status
// proves the charge never completed and fails the row as the legacy bulk sweep
// did. Any OTHER error (transport / 5xx / ambiguous) is NOT treated as a
// definitive failure — the charge may still have completed at Square, and
// marking the row failed would close the double-charge window (blocking a
// same-key retry with a 409) even though the money moved. Such rows stay
// pending for a later run.
func reconcileStalePaymentAtSquare(ctx context.Context, table, squarePaymentID string) staleReconcileResult {
pr, err := SquareClient.GetPayment(ctx, squarePaymentID)
if err != nil {
if squarePaymentErrorIsNotFound(err) {
log.Printf("Stale pending %s reconcile: Square payment %s not found (%v) — marking failed as the legacy sweep would", table, squarePaymentID, err)
return staleReconcileDefinitivelyFailed
}
log.Printf("Stale pending %s reconcile for Square payment %s hit an ambiguous error (%v) — leaving pending for a later sweep run", table, squarePaymentID, err)
return staleReconcileLeavePending
}
// Square's terminal payment states are COMPLETED, CANCELED, FAILED;
// APPROVED (authorization-only, delayed capture) and PENDING are
// NON-terminal — both can still transition to COMPLETED (via
// CompletePayment or delay_action=COMPLETE), so clawing back the funded
// gift card on either would risk reversing a charge that later lands.
// This app always creates payments with autocomplete (default true), so
// it never produces APPROVED/PENDING rows today, but the classification
// must match Square's documented state machine (librarian-verified
// 2026-05-20).
switch pr.Status {
case "COMPLETED":
return staleReconcileCompleted
case "CANCELED", "FAILED":
log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed
case "APPROVED", "PENDING":
log.Printf("Stale pending %s is %q at Square (non-terminal) — leaving pending for a later sweep run", table, pr.Status)
return staleReconcileLeavePending
default:
log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status)
return staleReconcileDefinitivelyFailed
}
}
// squarePaymentErrorIsNotFound reports whether a GetPayment error proves the
// payment does not exist at Square. The structured not-found check is primary
// (square.IsNotFound: NOT_FOUND code / HTTP 404 status / plain "HTTP 404" body);
// the message fallback covers only the dev mock's plain "payment not found"
// error, which carries no structured code.
func squarePaymentErrorIsNotFound(err error) bool {
if err == nil {
return false
}
if square.IsNotFound(err) {
return true
}
// Any other structured Square error code is authoritative — never
// substring-match its message.
if square.ErrorCode(err) != "" {
return false
}
msg := strings.ToUpper(err.Error())
return strings.Contains(msg, "NOT_FOUND") ||
strings.Contains(msg, "NOT FOUND")
}
// squareHasCode reports whether err carries a structured Square error code
// equal to any of codes. Errors without a structured code (the dev mock's
// plain errors) match nothing.
func squareHasCode(err error, codes ...string) bool {
if err == nil {
return false
}
code := square.ErrorCode(err)
for _, c := range codes {
if code == c {
return true
}
}
return false
}
// sweepSquarelessManualRefundsAtAttemptCap reconciles Square-less MANUAL refund
// rows that have already hit the attempt cap (maxManualRefundAttempts). The
// Square-less pre-pass inside sweepManualPendingSquareRefunds (refunds.go)
// filters refund_attempts < maxManualRefundAttempts, so a legacy manual refund
// on a payment with no square_payment_id that reached the cap (attempts
// incremented by pre-guard Square attempts) is never reconciled by it and would
// stay 'pending' forever, permanently blocking the over-refund guard (F10).
// Such rows can never be refunded via Square, so they are marked 'failed' and
// surfaced in the admin notification centre for in-person arrangement — the
// same terminal treatment the pre-pass gives rows under the cap. Returns the
// number of rows marked failed.
func sweepSquarelessManualRefundsAtAttemptCap(ctx context.Context) (int, error) {
rows, err := db.Conn.Query(ctx, fmt.Sprintf(`
UPDATE refunds r SET status = 'failed'
FROM payments p
WHERE p.id = r.payment_id
AND r.status = 'pending' AND r.refund_attempts >= %d
AND p.payment_method IN ('online_square', 'in_person_card')
AND p.square_payment_id IS NULL
AND r.origin = 'manual'
RETURNING r.id
`, maxManualRefundAttempts))
if err != nil {
return 0, err
}
defer rows.Close()
var failedIDs []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err == nil {
failedIDs = append(failedIDs, id)
}
}
if err := rows.Err(); err != nil {
return 0, err
}
if len(failedIDs) > 0 {
log.Printf("Marked %d Square-less manual refund(s) at the %d-attempt cap failed (in-person arrangement needed)", len(failedIDs), maxManualRefundAttempts)
}
insertRefundFailedNotifications(ctx, failedIDs)
return len(failedIDs), nil
}
// staleTerminalCheckoutAge is how old a still-pending terminal checkout must
// be before the sweep cancels it. Terminal checkouts normally complete within
// minutes; an hour is far past any legitimate card-reader interaction while
// still short enough that a completed charge can never be misread as stale.
const staleTerminalCheckoutAge = 1 * time.Hour
// SweepStaleTerminalCheckouts cancels terminal (card-machine) checkouts that
// are still PENDING/IN_PROGRESS long after they were created, so the terminal
// stops waiting on a customer who walked away. A checkout created by
// CreateTerminalPayment / CreateTillSale that is never polled would otherwise
// sit live at Square indefinitely; if it later completes it is an invisible,
// untracked charge.
//
// Two tables track live checkout IDs and are both swept:
// - terminal_checkouts: booking terminal checkouts created by
// CreateTerminalPayment. PENDING/IN_PROGRESS rows older than the cutoff
// are resolved at Square first: a checkout still waiting at Square is
// cancelled and re-checked once — if it completed during the cancel window
// the row is marked 'completed' (the poll handler records the payment),
// otherwise 'failed'; a COMPLETED checkout releases the in-flight guard
// (the poll handler records the payment); a definitively
// cancelled/expired checkout is marked 'failed'; an ambiguous status is
// left for a later run.
// - till_sales.square_checkout_id: card-machine till sales. A checkout still
// waiting at Square is cancelled and the sale marked 'failed' (the
// payment_status enum has no 'cancelled' value, and 'failed' is the same
// terminal state the stale-pending sweep uses, blocking the till
// pending-retry path); a checkout that completed during the cancel window
// (or was already COMPLETED when first checked) is RECORDED by the sweep —
// the sale is marked 'completed' with the returned square_payment_id, the
// till mirror of the booking path's recordUntrackedTerminalPayment (a
// never-polled sale would otherwise stay invisible in till reporting until
// the 24h blind-fail).
//
// Each row is checked at Square FIRST and only cancelled when the checkout is
// provably still waiting (ErrCheckoutPending): a COMPLETED checkout is never
// cancelled, and a checkout whose status is unknown (transport error) is left
// alone for a later run. After a cancel the checkout is re-checked once — the
// customer may have completed the payment in the cancel window, in which case
// the row is resolved to 'completed' rather than 'failed'.
func SweepStaleTerminalCheckouts(ctx context.Context) (int, error) {
cutoff := clock.Now().Add(-staleTerminalCheckoutAge)
var pending []staleTerminalCheckoutRow
// Booking terminal checkouts (CreateTerminalPayment) live in the
// terminal_checkouts table. A stale PENDING/IN_PROGRESS row means the
// checkout is still live at Square (or was left after a crash / lost poll).
rows, err := db.Conn.Query(ctx, `
SELECT 'terminal_checkout', checkout_id, checkout_id, booking_id
FROM terminal_checkouts
WHERE status IN ('PENDING', 'IN_PROGRESS')
AND created_at < $1
`, cutoff)
if err != nil {
return 0, err
}
for rows.Next() {
var r staleTerminalCheckoutRow
if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil {
log.Printf("Failed to scan stale terminal checkout row: %v", err)
continue
}
pending = append(pending, r)
}
rows.Close()
// Till-sale card-machine checkouts are tracked on the till_sales row.
rows, err = db.Conn.Query(ctx, `
SELECT 'till_sale', id, square_checkout_id, ''
FROM till_sales
WHERE status = 'pending' AND square_checkout_id IS NOT NULL
AND created_at < $1
`, cutoff)
if err != nil {
return 0, err
}
for rows.Next() {
var r staleTerminalCheckoutRow
if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil {
log.Printf("Failed to scan stale terminal checkout row: %v", err)
continue
}
pending = append(pending, r)
}
rows.Close()
resolved := 0
for _, r := range pending {
// An EMPTY checkout_id is a legacy provisional (pre-Square) row for
// which no Square call was ever possible — resolving it to failed
// directly is safe (R3).
if r.CheckoutID == "" {
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Provisional (pre-Square) terminal checkout row %s (%s) resolved as failed — no live checkout at Square", r.RowID, r.Kind)
continue
}
// A provisional row carries a synthetic "tmp-" checkout_id. It is no
// longer PROVABLY not live (H4): a hard crash between the
// terminal_checkouts insert and the provisional→real UPDATE leaves a
// LIVE checkout at Square (created under the idempotency key embedded
// in the tmp id) while the row still carries the synthetic id.
// Blind-failing would release the in-flight guard while that checkout
// can still complete at the terminal into an untracked charge. Query
// Square first and classify exactly like activeTerminalCheckoutID
// (handlers.go H4): COMPLETED → record the payment and release the
// guard; ErrCheckoutPending → the checkout is still live at Square,
// keep the guard; isTerminalCheckoutError (NOT_FOUND / CANCELED) →
// safe to fail (the crash happened before the Square call, or the
// checkout was cancelled); ambiguous → leave the row in flight. A tmp-
// id that never reached Square (or one Square cannot resolve) comes
// back NOT_FOUND — the expected outcome — and failing it is correct.
if strings.HasPrefix(r.CheckoutID, "tmp-") {
pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case gErr == nil && pr.Status == "COMPLETED":
// The checkout actually completed at the terminal. Record the
// payment if it was never polled/recorded — a COMPLETED charge
// must not stay an invisible untracked charge (H4). Booking
// checkouts record full payment rows; till-sale checkouts
// record the sale row.
if r.Kind == "terminal_checkout" {
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) {
resolved++
}
log.Printf("Provisional terminal checkout %s (%s %s) is COMPLETED at Square — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID)
case errors.Is(gErr, square.ErrCheckoutPending):
// The checkout is still live at Square — keep the in-flight
// guard so a second checkout cannot be created while it can
// still complete.
log.Printf("Provisional terminal checkout %s (%s %s) is still live at Square — leaving pending; the in-flight guard stays held", r.CheckoutID, r.Kind, r.RowID)
case isTerminalCheckoutError(gErr):
// NOT_FOUND (no checkout was ever created — the crash happened
// before the Square call) or CANCELED — safe to resolve failed.
switch {
case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr):
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
log.Printf("Provisional terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID)
case r.Kind == "till_sale":
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("CRITICAL: provisional terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID)
default:
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Provisional (pre-Square) terminal checkout row %q (%s %s) resolved as failed — no live checkout at Square", r.CheckoutID, r.Kind, r.RowID)
}
default:
// Ambiguous error — the checkout's money state at Square is
// unknown. Leave the row in flight rather than releasing the
// guard and allowing a second checkout.
log.Printf("Provisional terminal checkout %s (%s %s) status unknown (%v) — leaving pending for a later sweep", r.CheckoutID, r.Kind, r.RowID, gErr)
}
continue
}
// Conservative status check: only cancel a checkout that is provably
// still waiting at Square. A COMPLETED checkout must never be
// cancelled, and an ambiguous status (network error) is left alone for
// the next run.
pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case errors.Is(gErr, square.ErrCheckoutPending):
// Still live at the terminal — cancel it. A customer can complete
// the payment in the small window between the GetCheckout above and
// the cancel, so re-check once before marking the row failed: a
// COMPLETED charge must never be recorded as failed.
if cErr := SquareClient.CancelCheckout(ctx, r.CheckoutID); cErr != nil {
log.Printf("Failed to cancel stale terminal checkout %s (%s %s): %v", r.CheckoutID, r.Kind, r.RowID, cErr)
continue
}
recheck, rErr := SquareClient.GetCheckout(ctx, r.CheckoutID)
switch {
case rErr == nil && recheck.Status == "COMPLETED":
// The customer completed the payment during the cancel window.
// Record it if it was never polled/recorded — a COMPLETED
// checkout must not stay an untracked charge (H4). Booking
// checkouts record full payments rows; till-sale checkouts
// record the sale row (finding 3).
if r.Kind == "terminal_checkout" {
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, recheck) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, recheck) {
resolved++
}
log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) but re-check shows COMPLETED — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge)
case isTerminalCheckoutError(rErr) || errors.Is(rErr, square.ErrCheckoutPending):
// The cancel landed (CANCELED / cancel-requested / expired /
// still-reporting-pending-but-now-cancelled) — it can never
// complete, so resolve the row to the terminal 'failed' state.
// A till sale's funded gift card is clawed back (a cancel that
// landed cannot later complete); a booking terminal_checkout
// has no gift card.
if r.Kind == "till_sale" {
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
} else if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) — marked failed", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge)
default:
// The cancel succeeded but the re-check itself is ambiguous —
// leave the row for a later run.
log.Printf("Terminal checkout %s was cancelled but its re-check is ambiguous (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, rErr, r.Kind, r.RowID)
}
case gErr == nil && pr.Status == "COMPLETED":
if r.Kind == "terminal_checkout" {
// The checkout completed at Square but was never polled/recorded
// (the frontend never called GetCheckoutStatus). Record the
// payment rows now — otherwise the charge stays invisible to
// refunds and TotalPaid (H4).
if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) {
resolved++
}
} else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) {
// A never-polled COMPLETED till-sale checkout must not stay
// pending until the 24h blind-fail (finding 3) — the charge is
// real, so the sale is recorded completed with the payment id.
resolved++
}
case isTerminalCheckoutError(gErr):
// The checkout is cancelled / cancel-requested / expired at Square.
switch {
case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr):
// The error PROVES the checkout can never complete (CANCELED /
// NOT_FOUND, and no bare CANCEL_REQUESTED) — claw back the
// funded gift card along with the failed mark.
if clawBackTillSaleFunding(ctx, r.RowID) {
resolved++
}
log.Printf("Terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID)
case r.Kind == "till_sale":
// CANCEL_REQUESTED-only: Square does not promise non-completion
// in that state, so the charge may still land. Mark the sale
// failed but DO NOT claw back the funded gift card.
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("CRITICAL: terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID)
default:
if markTerminalCheckoutRowFailed(ctx, r) {
resolved++
}
log.Printf("Terminal checkout %s is definitively terminal (%v) — marked %s %s failed", r.CheckoutID, gErr, r.Kind, r.RowID)
}
default:
// Ambiguous transport/unknown status — leave for a later run.
log.Printf("Terminal checkout %s status unknown (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, gErr, r.Kind, r.RowID)
}
}
return resolved, nil
}
// staleTerminalCheckoutRow is one live-checkout row the sweep reads from
// either table so it can resolve the checkout at Square before touching the
// row. Kind is "terminal_checkout" (booking, terminal_checkouts table) or
// "till_sale" (till_sales.square_checkout_id). BookingID is the booking the
// terminal_checkouts row belongs to ("" for till sales) — needed to record an
// untracked COMPLETED terminal charge (H4).
type staleTerminalCheckoutRow struct {
Kind string
RowID string
CheckoutID string
BookingID string
}
// markTerminalCheckoutRowFailed moves one tracked row to the terminal 'failed'
// state after its checkout is cancelled or proven terminal at Square. Returns
// true when the row was updated (status was still active).
func markTerminalCheckoutRowFailed(ctx context.Context, r staleTerminalCheckoutRow) bool {
var table, where string
if r.Kind == "till_sale" {
table = "till_sales"
where = "id = $1 AND status = 'pending'"
} else {
table = "terminal_checkouts"
where = "checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')"
}
// table is an internal constant ("till_sales"/"terminal_checkouts"), never
// user input, but the identifier is routed through pgx.Identifier.Sanitize
// so no raw, unquoted table name is ever concatenated into the statement.
tag, err := db.Conn.Exec(ctx, `
UPDATE `+pgx.Identifier{table}.Sanitize()+` SET status = 'failed', updated_at = NOW()
WHERE `+where, r.RowID)
if err != nil {
log.Printf("Failed to mark %s %s failed: %v", r.Kind, r.RowID, err)
return false
}
return int(tag.RowsAffected()) > 0
}
// recordUntrackedTerminalPayment records the payments row(s) for a terminal
// checkout that COMPLETED at Square but was never polled/recorded, then marks
// the terminal_checkouts row COMPLETED. The stale sweep otherwise leaves a real
// charge with NO payments row — invisible to refunds and TotalPaid (H4). The
// money-recording work lives in the SHARED recordTerminalPaymentTx core (also
// used by the GetCheckoutStatus poll handler), so both writers run the same
// dedup / FOR UPDATE recheck / insert / M4 split / VAT / guarded checkout
// update; this wrapper supplies the sweep-specific pieces: the advisory lock,
// the cancelled-booking critical admin notification and the CRITICAL completion
// log. Returns true when the checkout row was resolved (payment recorded,
// already recorded, or refused on a cancelled booking); false when recording
// failed (the row is left pending so the next sweep re-runs the whole
// reconcile).
func recordUntrackedTerminalPayment(ctx context.Context, checkoutID, bookingID string, pr *square.PaymentResult) bool {
if pr == nil || pr.SquarePayID == "" {
log.Printf("CRITICAL: terminal checkout %s is COMPLETED at Square but carries no Square payment ID — cannot record the payment — MANUAL RECONCILIATION REQUIRED", checkoutID)
return false
}
// Serialize with the poll handler (GetCheckoutStatus): both record the same
// Square payment, so the advisory lock + dedup SELECT prevent a double
// insert (the idempotency_key UNIQUE constraint is the backstop).
pinConn, err := db.Conn.Acquire(ctx)
if err != nil {
log.Printf("Failed to acquire connection for terminal-completion lock: %v", err)
return false
}
defer pinConn.Release()
lockOK, err := acquireAdvisoryLock(ctx, pinConn, "crussell:terminal:"+pr.SquarePayID)
if err != nil {
log.Printf("Failed to acquire terminal-completion serialization lock for %s: %v", pr.SquarePayID, err)
return false
}
if !lockOK {
log.Printf("Terminal-completion serialization lock for %s not acquired within bound — a poll is already recording this checkout", pr.SquarePayID)
return false
}
defer releasePaymentLock(pinConn, "crussell:terminal:"+pr.SquarePayID)
tx, err := db.Conn.Begin(ctx)
if err != nil {
log.Printf("Failed to begin terminal-completion transaction: %v", err)
return false
}
defer func() {
if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
log.Printf("Failed to rollback terminal-completion transaction: %v", err)
}
}()
paymentID, recErr := recordTerminalPaymentTx(ctx, tx, checkoutID, bookingID, pr)
if recErr != nil {
if errors.Is(recErr, errTerminalBookingNotPayable) {
// Sweep-only behaviour: the poll surfaces a 409 to a live caller,
// but a sweep has no one to tell, so alert ops in the admin
// notification centre that money was taken at Square and MUST be
// refunded manually (the core already committed the checkout's
// 'failed' mark).
insertCriticalPaymentNotification(ctx, &bookingID, nil)
return true
}
// The core logged the failure; the row is left pending so the next
// sweep run re-runs the whole reconcile.
return false
}
log.Printf("CRITICAL: recorded untracked terminal charge %s (booking %s) from stale checkout %s — payment row %s created (never polled by the frontend)", pr.SquarePayID, bookingID, checkoutID, paymentID)
return true
}
// errTerminalBookingNotPayable is returned by recordTerminalPaymentTx when the
// booking moved out of a payable state (cancelled / lapsed / no-show) between
// the terminal charge completing at Square and the record attempt. The core has
// ALREADY committed the terminal_checkouts 'failed' mark (and a non-payable
// commit failure is wrapped with this sentinel too), so the caller must not
// roll back: the poll surfaces a 409 conflict, the sweep inserts the critical
// admin notification and resolves the row.
var errTerminalBookingNotPayable = errors.New("booking is no longer payable for a terminal payment")
// recordTerminalPaymentTx records a COMPLETED terminal checkout as payments
// rows, applying the M4 tip split, per-record VAT and booking completion. Both
// the GetCheckoutStatus poll handler and the stale-terminal sweep call it so
// the money-recording logic exists exactly once.
//
// CALL-SITE CONTRACT: both call sites must keep passing the SAME inputs — the
// caller's transaction, the terminal_checkouts checkout_id, the booking id and
// the *square.PaymentResult returned by GetCheckout — and each must acquire the
// advisory lock on "crussell:terminal:<SquarePayID>" on its own pinned pool
// connection BEFORE calling (the core runs inside the caller's transaction).
// The core commits the transaction and completes a now-fully-paid booking; the
// caller owns only the error mapping (errTerminalBookingNotPayable vs generic),
// the HTTP response / sweep return value and any post-commit behaviour (e.g.
// the sweep's critical notification).
func recordTerminalPaymentTx(ctx context.Context, tx pgx.Tx, checkoutID, bookingID string, pr *square.PaymentResult) (string, error) {
service := NewPaymentService()
// Dedup by Square payment ID: a concurrent poll (or a prior sweep run)
// already recorded this charge — release the in-flight guard and return the
// existing row so the caller reports the same payment id. The checkout is
// marked COMPLETED under the same guarded status predicate used everywhere
// else, so a row already resolved to 'failed' is never resurrected.
var existingID string
if err := tx.QueryRow(ctx, `
SELECT id FROM payments
WHERE booking_id = $1 AND square_payment_id = $2
`, bookingID, pr.SquarePayID).Scan(&existingID); err == nil {
if _, upErr := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); upErr != nil {
slog.Error("Failed to mark terminal checkout completed on dedup", "checkout_id", checkoutID, "err", upErr)
return "", fmt.Errorf("mark terminal checkout %s completed: %w", checkoutID, upErr)
}
if cErr := tx.Commit(ctx); cErr != nil {
slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", cErr)
return "", cErr
}
return existingID, nil
} else if !errors.Is(err, pgx.ErrNoRows) {
slog.Error("Failed to check for existing terminal payment", "checkout_id", checkoutID, "booking_id", bookingID, "square_payment_id", pr.SquarePayID, "err", err)
return "", err
}
// Re-check the booking status under the advisory lock (mirrors
// GetCheckoutStatus): a cancellation/eviction that committed between the
// terminal charge completing at Square and this record attempt must not
// produce a completed payment on a cancelled/lapsed/no-show booking — the
// cancellation refund path computes refunds from completed payments and
// would silently exclude this charge. Mark the checkout failed and alert
// ops: money was taken at Square and MUST be refunded manually.
var recheckStatus string
// FOR UPDATE (C5): serializes against the cancellation path's lock on the
// same row so a concurrent cancellation cannot commit between this recheck
// and the transaction commit below.
if err := tx.QueryRow(ctx, `SELECT status FROM bookings WHERE id = $1 FOR UPDATE`, bookingID).Scan(&recheckStatus); err != nil {
slog.Error("CRITICAL: Square payment was processed but re-reading booking status failed — manual reconciliation required", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "err", err)
return "", err
}
if !bookingStatusAllowsCompletedPayment(recheckStatus) {
slog.Error("CRITICAL: Square payment was processed but booking is no longer payable — marking checkout failed; money taken at Square MUST be refunded manually", "square_payment_id", pr.SquarePayID, "checkout_id", checkoutID, "booking_id", bookingID, "status", recheckStatus)
if _, upErr := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'failed', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); upErr != nil {
slog.Error("CRITICAL: Square payment landed on a non-payable booking but marking checkout failed errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", upErr)
}
if cErr := tx.Commit(ctx); cErr != nil {
slog.Error("CRITICAL: Square payment landed on a non-payable booking and committing the checkout-failed mark errored — manual reconciliation required", "square_payment_id", pr.SquarePayID, "status", recheckStatus, "booking_id", bookingID, "checkout_id", checkoutID, "err", cErr)
// Keep the sentinel wrapped so BOTH callers still surface the
// conflict path (poll 409 / sweep notification) even when the
// failed-mark commit itself failed.
return "", fmt.Errorf("%w: %v", errTerminalBookingNotPayable, cErr)
}
return "", errTerminalBookingNotPayable
}
// The payment type the admin charged is recorded on the checkout row by
// CreateTerminalPayment; fall back to 'full' for legacy rows. tip_enabled
// records whether the customer EXPLICITLY requested a tip (the frontend's
// tip_enabled flag): an overflow beyond the remaining booking value must
// only be recorded as a tip record when the customer asked for one (B3) —
// an accidental overpayment must never be relabelled gratuity.
var checkoutPaymentType string
var checkoutTipEnabled bool
if err := tx.QueryRow(ctx, `
SELECT payment_type, tip_enabled FROM terminal_checkouts WHERE checkout_id = $1
`, checkoutID).Scan(&checkoutPaymentType, &checkoutTipEnabled); err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
slog.Error("Failed to read payment type for checkout", "checkout_id", checkoutID, "err", err)
}
checkoutPaymentType = "full"
}
idempotencyKey := bookingID + "-terminal-" + strconv.FormatInt(pr.Amount, 10) + "-" + pr.SquarePayID
record := PaymentRecord{
BookingID: bookingID,
PaymentType: checkoutPaymentType,
PaymentMethod: "in_person_card",
Status: "completed",
Amount: float64(pr.Amount) / 100.0,
SquarePaymentID: &pr.SquarePayID,
IdempotencyKey: &idempotencyKey,
Fees: float64(pr.Fees) / 100.0,
CreatedAt: clock.Now(),
UpdatedAt: clock.Now(),
}
// The booking user is read here (before the M4 carve) because the carve
// computes the campaign discounts applyEligibleCampaignsAtPayment is about
// to apply below, and that apply needs the payer's user id.
var bookingUserID string
if err := tx.QueryRow(ctx, `SELECT COALESCE(user_id, '') FROM bookings WHERE id = $1`, bookingID).Scan(&bookingUserID); err != nil {
slog.Error("Failed to load booking user for terminal campaign apply", "booking_id", bookingID, "err", err)
}
// M4 split: a terminal charge above the remaining booking value is a tip —
// record it as its own record so only the booking portion is refundable.
// B3: only when the customer EXPLICITLY requested a tip (checkout
// tip_enabled). Without an explicit tip request the overflow is an
// accidental overpayment, not gratuity: it stays on the booking record at
// the full charged amount (the excess is refundable, never a tip). The
// checkout-creation side clamps the amount to the remaining value when no
// tip is requested, so this branch only fires for legacy/edge checkouts.
var records []PaymentRecord
bookingInfo, bErr := service.GetBookingPaymentInfo(ctx, bookingID)
if bErr == nil && bookingInfo != nil {
charged := float64(pr.Amount) / 100.0
remainingBookingValue := math.Max(0, bookingInfo.TotalAmount-bookingInfo.TotalPaid)
// Reserve the headroom of the campaign discounts that
// applyEligibleCampaignsAtPayment is about to mint: a
// terminal charge already priced to the discounted amount (the frontend
// charges totaldiscount plus any explicit tip) must carve the tip
// against the DISCOUNTED obligation, not the full total — otherwise the
// whole charge lands as booking portion and the tip is silently
// absorbed into deposit/balance instead of being recorded as gratuity
// (M4). The no-tip case is unchanged: the booking portion exactly
// covers the discounted obligation and the discount fills the rest.
if pending := pendingCampaignDiscountAmount(ctx, tx, bookingID, bookingUserID, bookingInfo.TotalAmount); pending > 0.004 {
remainingBookingValue = math.Max(0, remainingBookingValue-pending)
}
bookingPortion := math.Min(charged, remainingBookingValue)
bookingPortion = math.Round(bookingPortion*100) / 100
tipAmount := math.Max(0, math.Round((charged-bookingPortion)*100)/100)
if checkoutTipEnabled && tipAmount > 0.004 {
records = buildTerminalSplitRecords(record, bookingInfo, bookingPortion, tipAmount)
}
}
hasTip := false
for _, rec := range records {
if rec.PaymentType == "tip" {
hasTip = true
break
}
}
// Apply eligible campaigns BEFORE the records are inserted when the M4
// carve minted a TIP record and a campaign discount is pending. The apply
// path refuses discounts once a booking has 2+ completed real payments, and
// the split records (deposit + balance + tip) would count as 2+ even though
// they are ONE charge — refusing the discount the frontend already priced
// in would leave the booking under-discounted and never complete. Applying
// before the insert is money-safe: a tip record only exists when the charge
// exceeds the DISCOUNTED obligation, so the booking portion plus the
// pending discount can never exceed the total — the F1 full-amount
// over-credit cannot occur here (a full-amount charge never produces a tip
// record, so it keeps the after-insert apply whose cap refuses it).
appliedCampaignsBeforeInsert := false
if hasTip && bookingInfo != nil && bErr == nil {
if pendingCampaignDiscountAmount(ctx, tx, bookingID, bookingUserID, bookingInfo.TotalAmount) > 0.004 {
if applyErr := applyEligibleCampaignsAtPayment(ctx, tx, bookingID, bookingUserID, nil); applyErr != nil {
slog.Error("Failed to apply eligible campaigns before terminal payment insert", "booking_id", bookingID, "err", applyErr)
}
appliedCampaignsBeforeInsert = true
}
}
if len(records) == 0 {
records = []PaymentRecord{record}
}
primary := records[0]
paymentID, err := service.CreatePaymentRecordTx(ctx, tx, primary, nil)
if err != nil {
slog.Error("Failed to create payment record for terminal charge", "square_payment_id", pr.SquarePayID, "err", err)
return "", err
}
ApplyVATToBookingPayment(ctx, tx, paymentID)
for _, rec := range records[1:] {
pid, cErr := service.CreatePaymentRecordTx(ctx, tx, rec, nil)
if cErr != nil {
slog.Error("Failed to create terminal tip split record", "square_payment_id", pr.SquarePayID, "err", cErr)
return "", cErr
}
ApplyVATToBookingPayment(ctx, tx, pid)
}
// B13: apply eligible campaign discounts at record time, inside the same
// transaction as the payment insert. The card-machine checkout completes
// ASYNCHRONOUSLY (the GetCheckoutStatus poll or the stale-checkout sweep),
// so there is no HTTP response to surface an exhausted-at-apply campaign
// the way the synchronous saved-card/booking paths do — a campaign that
// exhausts between the frontend's preview and this record stays a silent
// skip (the customer was already charged the discounted amount the frontend
// showed; the merchant absorbs the shortfall — documented limitation of the
// async flow). An AVAILABLE campaign MUST be applied here rather than
// deferred to the completion side-effects: completeFullyPaidBooking only
// runs ApplyBookingCompletionSideEffects when bookingIsFullyPaid (real
// money covers the total), at which point capDiscountToRemainingObligation
// sees zero headroom — so a discounted-amount terminal charge would never
// get its discount row and never complete. Applying here (with the same
// capDiscountToRemainingObligation guard as every other path) mints the
// discount row so bookingIsFullyPaid sees it; the completion side-effects
// skip re-application via their already-recorded guards. The apply runs
// AFTER the payment record insert so the headroom counts the charge as real
// money (never over-credits), and after the M4 tip split so tip records
// never affect the discount computation. When the M4 carve minted a tip
// record the discount was already applied BEFORE the insert (see above), so
// the apply is skipped here — a second run is idempotent but unnecessary.
// The booking user was already read above for the M4 carve.
if !appliedCampaignsBeforeInsert {
if applyErr := applyEligibleCampaignsAtPayment(ctx, tx, bookingID, bookingUserID, nil); applyErr != nil {
slog.Error("Failed to apply eligible campaigns for terminal checkout", "booking_id", bookingID, "err", applyErr)
}
}
// Release the in-flight guard: this checkout is now recorded.
if _, err := tx.Exec(ctx, `
UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW()
WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')
`, checkoutID); err != nil {
slog.Error("Failed to mark terminal checkout completed", "checkout_id", checkoutID, "err", err)
return "", err
}
if err := tx.Commit(ctx); err != nil {
slog.Error("Failed to commit terminal-completion transaction", "checkout_id", checkoutID, "err", err)
return "", err
}
// The booking may now be fully paid — complete it like the poll handler does.
completeFullyPaidBooking(ctx, bookingID)
return paymentID, nil
}
// pendingCampaignDiscountAmount sums the campaign discounts that
// applyEligibleCampaignsAtPayment is about to apply for the booking (its
// 'expected' argument is nil in the terminal flow, so it recomputes the
// eligible set from scratch). The discount rows do not exist yet when the M4
// tip carve runs — they are minted later in the same transaction — so the
// carve must reserve their headroom NOW or a terminal charge already priced to
// the discounted amount absorbs the customer's explicit tip into the booking
// portion (M4). Referral discounts are excluded: the terminal apply path never
// mints them.
func pendingCampaignDiscountAmount(ctx context.Context, q db.Querier, bookingID, userID string, bookingTotal float64) float64 {
var total float64
for _, d := range ComputeEligibleDiscounts(ctx, q, bookingID, userID, bookingTotal) {
if d.Source == "campaign" {
total += d.Amount
}
}
return math.Round(total*100) / 100
}
// recordUntrackedTillSalePayment records a stale card-machine till sale whose
// checkout COMPLETED at Square but was never polled/recorded: the sale is
// marked 'completed' with the returned square_payment_id written back. A
// never-polled COMPLETED checkout would otherwise leave the sale pending until
// the 24h blind-fail — the charge is real but invisible in till reporting
// (money-safe: the gift card was funded pre-charge, a blind-fail never claws
// back, and failed rows reject retries — but the till reporting would be wrong,
// finding 3). There is no split/deposit concept for till sales — a single
// completed row, exactly the update the till poll handler (GetTillCheckoutStatus)
// performs. The WHERE status = 'pending' guard makes it safe against a
// concurrent poll: only one of them wins the update, the loser sees 0 rows and
// backs off. Returns true when the sale was updated; false when the checkout
// carries no Square payment id (the sale is left PENDING — never blind-failed
// — with a CRITICAL log for manual reconciliation) or the sale was already
// resolved by someone else.
func recordUntrackedTillSalePayment(ctx context.Context, saleID string, pr *square.PaymentResult) bool {
if pr == nil || pr.SquarePayID == "" {
log.Printf("CRITICAL: terminal checkout is COMPLETED at Square but carries no Square payment ID — cannot record till sale %s — leaving it PENDING — MANUAL RECONCILIATION REQUIRED", saleID)
return false
}
tag, err := db.Conn.Exec(ctx, `
UPDATE till_sales SET status = 'completed', square_payment_id = $1, updated_at = NOW()
WHERE id = $2 AND status = 'pending'
`, pr.SquarePayID, saleID)
if err != nil {
log.Printf("Failed to record untracked terminal till sale %s as completed: %v", saleID, err)
return false
}
if int(tag.RowsAffected()) == 0 {
// A concurrent poll (or a prior sweep run) already recorded the sale —
// nothing left for this run to do.
log.Printf("Terminal till sale %s was already resolved — skipping untracked terminal completion record", saleID)
return false
}
log.Printf("CRITICAL: recorded untracked terminal till-sale charge %s (sale %s) from a stale checkout — sale marked completed (never polled by the frontend)", pr.SquarePayID, saleID)
return true
}
// isTerminalCheckoutError reports whether a GetCheckout error proves the
// checkout can never complete. Square's HTTP client returns ErrCheckoutPending
// for a still-live checkout and surfaces a definitively CANCELED status as a
// "square: checkout <id> is CANCELED (not COMPLETED)" error; an expired
// checkout returns a structured NOT_FOUND API error (the mock uses a plain
// "checkout not found"). Any other error (timeout, 5xx) leaves the money state
// ambiguous, so the checkout must stay in flight.
//
// Structured codes are authoritative when present: NOT_FOUND (via
// square.IsNotFound) and an explicit CANCELED / CANCEL_REQUESTED code classify
// as terminal. The formatted-message match applies only to errors that carry
// no structured code — the dev mock's plain errors and the real client's
// client-side "is <status> (not COMPLETED)" error, which it synthesizes
// without a squareAPIError.
func isTerminalCheckoutError(err error) bool {
if err == nil || errors.Is(err, square.ErrCheckoutPending) {
return false
}
if square.IsNotFound(err) || squareHasCode(err, "CANCELED", "CANCEL_REQUESTED") {
return true
}
// Any other structured Square error code is authoritative — never
// substring-match its message.
if square.ErrorCode(err) != "" {
return false
}
msg := strings.ToUpper(err.Error())
return strings.Contains(msg, "CANCELED") ||
strings.Contains(msg, "CANCEL_REQUESTED") ||
strings.Contains(msg, "NOT_FOUND") ||
strings.Contains(msg, "NOT FOUND")
}
// isCheckoutDefinitivelyDead reports whether a GetCheckout error PROVES the
// checkout can never complete — the condition under which a till sale's funded
// gift card may be clawed back. It is stricter than isTerminalCheckoutError:
// a CANCEL_REQUESTED-only classification (Square does not promise non-completion
// in that state) is NOT definitive proof, so the charge may still land and the
// funding must stay put. Only an explicit CANCELED or NOT_FOUND status is
// definitive; a CANCEL_REQUESTED message counts only when it ALSO carries
// CANCELED.
//
// Structured codes are authoritative when present: CANCELED and NOT_FOUND are
// definitive, a bare CANCEL_REQUESTED code is not. The formatted-message match
// applies only to errors that carry no structured code (dev mock plain errors,
// and the real client's client-side "is <status> (not COMPLETED)" error).
func isCheckoutDefinitivelyDead(err error) bool {
if err == nil || errors.Is(err, square.ErrCheckoutPending) {
return false
}
if code := square.ErrorCode(err); code != "" {
return code == "CANCELED" || code == "NOT_FOUND"
}
// Non-structured errors: an HTTP 404 / plain 404 body is definitive
// (IsNotFound); the message match covers the client-side CANCELED status
// error and the mock's plain not-found errors.
if square.IsNotFound(err) {
return true
}
msg := strings.ToUpper(err.Error())
if !strings.Contains(msg, "CANCELED") && !strings.Contains(msg, "NOT_FOUND") && !strings.Contains(msg, "NOT FOUND") {
return false
}
if strings.Contains(msg, "CANCEL_REQUESTED") && !strings.Contains(msg, "CANCELED") {
return false
}
return true
}
// clawBackTillSaleFunding reverts the gift-card funding of a till sale whose
// card-machine checkout is provably dead, marking the sale failed at the same
// time. The terminal-checkout sweep's rows carry no gift-card context, so the
// sale's item/amount/redeem target are looked up here (is_create via the
// created_at equality with the gift card) and handed to the claim-first
// revertGiftCardFunding. A sale with no gift card (future retail product /
// orphaned item) is only marked failed. Returns true when the sale was
// resolved to failed; false when it was already resolved by someone else, had
// no gift card, or the clawback failed (CRITICAL logged).
func clawBackTillSaleFunding(ctx context.Context, tillSaleID string) bool {
var itemType string
var itemID sql.NullString
var totalAmount float64
var redeemedBy sql.NullString
var isCreate *bool
err := db.Conn.QueryRow(ctx, `
SELECT ts.item_type, ts.item_id, ts.total_amount, gc.redeemed_by,
(ts.created_at = gc.created_at) AS is_create
FROM till_sales ts
LEFT JOIN gift_cards gc ON gc.id = ts.item_id
WHERE ts.id = $1
`, tillSaleID).Scan(&itemType, &itemID, &totalAmount, &redeemedBy, &isCreate)
if err != nil {
log.Printf("CRITICAL: failed to look up till sale %s for funding clawback: %v — MANUAL RECONCILIATION REQUIRED", tillSaleID, err)
return false
}
if itemType != "gift_card" || !itemID.Valid || itemID.String == "" || isCreate == nil {
// No gift card to claw back — mark the sale failed without touching
// any card.
tag, upErr := db.Conn.Exec(ctx, `
UPDATE till_sales SET status = 'failed', updated_at = NOW()
WHERE id = $1 AND status = 'pending'
`, tillSaleID)
if upErr != nil {
log.Printf("Failed to mark till sale %s failed: %v", tillSaleID, upErr)
return false
}
return int(tag.RowsAffected()) > 0
}
action := "topup"
if *isCreate {
action = "create"
}
var redeem *string
if redeemedBy.Valid && redeemedBy.String != "" {
redeem = &redeemedBy.String
}
if revErr := revertGiftCardFunding(ctx, action, itemID.String, totalAmount, redeem, tillSaleID); revErr != nil {
if errors.Is(revErr, errTillSaleNotPending) {
log.Printf("Till sale %s was already resolved (not pending) — skipping funding clawback", tillSaleID)
return false
}
log.Printf("CRITICAL: funding clawback for till sale %s (gift card %s) failed: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", tillSaleID, itemID.String, revErr)
return false
}
return true
}