package payments import ( "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" "log" "math" "strconv" "strings" "time" "crussell/clock" "crussell/db" "crussell/internal/square" "github.com/jackc/pgx/v5" ) // SweepStalePendingPayments resolves pending payment records that are older // than Square's idempotency-key retention window (~24h). A pending record // means the DB committed but the Square charge outcome is unknown; it // normally resolves on a same-key client retry. But if the client abandoned // the attempt, the record stays pending forever — and retrying it after the // key expires would ISSUE A SECOND CHARGE (Square no longer dedups). Failing // stale pendings closes that double-charge window: a late retry finds a // 'failed' record and stops instead of charging again. // // Before failing a row, the sweep reconciles it against Square: a // genuinely-charged row (Square success, DB post-charge failure) with a // square_payment_id is rescued to 'completed' instead of being swept to // 'failed' with no automatic resolution — the money would otherwise be lost // in limbo (MINOR-R3). Reconciliation is deliberately minimal: status + // updated_at only, no split/VAT recomputation (that is the handler's job; the // row is >24h stale and this is a reconciliation rescue). A row with NO // square_payment_id but a STORED idempotency key (the lost-response case — the // charge may have completed at Square with the response never received) is // reconciled at Square by replaying the key (ReplayPaymentByKey) at an EARLIER // 22h cutoff, while the key is still inside Square's ~24h retention window: a // COMPLETED charge is rescued to 'completed' with the real square_payment_id // written back, and a charge Square proves never happened is failed. Rows with // neither a square_payment_id nor a stored key cannot be reconciled and are // failed with a WARN exactly as the legacy sweep did. // // Only online/till card payments can be pending — cash/giftcard/on_the_house // are committed synchronously and never enter this state. Both the payments // table and till_sales carry pending card-sale rows and are swept here. const stalePendingPaymentAge = 24 * time.Hour // stalePendingKeyedAge is how old a pending row with a stored idempotency key // (but no square_payment_id — the lost-response case) must be before the sweep // reconciles it at Square by replaying the key. It is deliberately 2h EARLIER // than stalePendingPaymentAge so the replay lands comfortably inside Square's // ~24h idempotency-key retention window: replaying at exactly 24h risks the key // already having expired, and an expired key would make the probe rejection // look like "never charged" even when the charge actually landed. Rows older // than stalePendingPaymentAge when swept can no longer be replayed // trustworthily and fall back to the legacy blind-fail + WARN. const stalePendingKeyedAge = 22 * time.Hour func SweepStalePendingPayments(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-stalePendingPaymentAge) keyedCutoff := clock.Now().Add(-stalePendingKeyedAge) // Pass 1 (earlier cutoff): rows with a stored idempotency key but NO // square_payment_id are the lost-response case — the charge may have landed // at Square with the response lost. They are reconciled at Square by // replaying the key (ReplayPaymentByKey) while the key is still inside // Square's ~24h retention window: a COMPLETED charge is rescued to // 'completed' with the real square_payment_id, a charge Square proves never // happened is failed (and a till sale's funded gift card clawed back), an // ambiguous answer leaves the row pending for the next run, and a row // already past the retention window is blind-failed with a WARN exactly as // the legacy sweep did (replaying an expired key would misread the probe // rejection as "never charged"). payKeyedCount, payKeyedCompleted, payKeyedUnverified, err := sweepKeyedStaleRows(ctx, "payments", keyedCutoff) if err != nil { return 0, err } tillKeyedCount, tillKeyedCompleted, tillKeyedUnverified, err := sweepKeyedStaleRows(ctx, "till_sales", keyedCutoff) if err != nil { return 0, err } // Pass 2 (legacy 24h cutoff): rows WITH a square_payment_id are reconciled // by payment id; rows with neither a payment id nor a stored key cannot be // reconciled and are failed directly (WARN — the charge outcome is unknown). payCount, payCompleted, err := sweepStaleRows(ctx, "payments", cutoff) if err != nil { return 0, err } tillCount, tillCompleted, err := sweepStaleRows(ctx, "till_sales", cutoff) if err != nil { return 0, err } total := payKeyedCount + tillKeyedCount + payCount + tillCount payTotal := payKeyedCount + payCount tillTotal := tillKeyedCount + tillCount completed := payKeyedCompleted + tillKeyedCompleted + payCompleted + tillCompleted if total > 0 { log.Printf("[SWEEP] Resolved %d stale pending payments (%d payments, %d till sales) older than %s — late retries will be rejected, preventing a second Square charge; %d reconciled to completed against Square (%d payments, %d till sales)", total, payTotal, tillTotal, stalePendingPaymentAge, completed, payKeyedCompleted+payCompleted, tillKeyedCompleted+tillCompleted) } // Routine sweep bookkeeping, not an incident: failing stale pending rows is // the sweep's DESIGNED behaviour. A row swept to failed may still have been // charged at Square with a lost response, so note it at WARN level for an // admin doing a periodic money reconciliation — but this fires on every // normal run and must not be elevated to CRITICAL (which is reserved for // genuinely unrecoverable post-charge branches). Rows the keyed reconcile // PROVED never charged are deliberately excluded from the WARN (they cannot // have moved money); only rows failed without a reconcile proof — keyless // rows and keyed rows past the retention window — are counted. if payKeyedUnverified > 0 { log.Printf("[SWEEP] WARN: %d pending payments with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response — verify before refunding/charging", payKeyedUnverified) } if tillKeyedUnverified > 0 { log.Printf("[SWEEP] WARN: %d pending till sales with a stored idempotency key but no square_payment_id were marked failed without a replay reconcile (key retention window already closed) — may have been charged at Square with a lost response", tillKeyedUnverified) } if payCount > 0 { log.Printf("[SWEEP] WARN: %d pending payments marked failed may have been charged at Square with a lost response — verify before refunding/charging", payCount-payCompleted) } if tillCount > 0 { log.Printf("[SWEEP] WARN: %d pending till sales marked failed may have been charged at Square with a lost response", tillCount-tillCompleted) } return total, nil } // staleRow is one stale pending row read by the sweep so it can reconcile // rows that carry a Square reference BEFORE failing them. For till_sales rows // the gift-card context needed to claw back funded money on a definitive // failure is carried alongside: the card id, who it was redeemed to, whether // this sale created the card (created_at equality — provable because both // timestamps are the transaction-start NOW() in the same tx), and the sale // amount (the exact funding this sale added). type staleRow struct { ID string SquarePaymentID string // IdempotencyKey is the deterministic Square idempotency key stored on the // pending row ("" when absent). Rows WITH a key but NO square_payment_id are // the lost-response case: the sweep replays the key at Square to learn the // true charge outcome before declaring failure. IdempotencyKey string // SquareSourceID is the exact source_id sent in the original CreatePayment // call, stored on the row so the replay-by-key can rebuild an IDENTICAL // request body (same key + source + amount). Replaying a different body // would return IDEMPOTENCY_KEY_REUSED, which proves nothing about whether // the charge landed. SquareSourceID string // SquareRequestSnapshot is the verbatim original CreatePayment request JSON // stored on the row at charge time (square_request_snapshot) — the FULL // body the replay-by-key must repeat (source, key, amount, customer_id, // reference_id, note, buyer_email_address, verification_token, ...). // Square's idempotency dedup compares the whole request, so a replay built // from partial row data returns IDEMPOTENCY_KEY_REUSED for a RETAINED key // and the row stays pending forever (safe but never auto-rescued). Nil for // legacy rows — the sweep then rebuilds the minimal body from the stored // key + source + amount. SquareRequestSnapshot []byte // AmountPence is the row's charge amount in pence — the amount the original // CreatePayment used. The replay-by-key must repeat it so Square's // idempotency dedup returns the original payment. AmountPence int64 // CreatedAt is the pending row's creation time; rows already past Square's // idempotency-key retention window cannot be replayed trustworthily. CreatedAt time.Time // BookingID is the payments row's booking_id ("" when the row has none). // A payments row with NO booking is a gift-card purchase (BuyGiftCard // inserts without a booking): rescuing it to 'completed' on a Square // COMPLETED reconcile would permanently block the same-key retry that // delivers the card, so such rows are kept pending instead (C6). BookingID *string // CreatedBy is the payments row's created_by user id (gift-card purchases // always carry the purchaser), used to attribute the critical-payment admin // notification. CreatedBy *string ItemID string // till_sales.item_id — the gift card ("" when NULL / non-gift-card) RedeemToUserID *string // gift_cards.redeemed_by — user credited by a create-with-redeem IsCreate bool // true when this sale created the gift card (timestamps equal) HasGiftCard bool // false when the LEFT JOIN found no gift_cards row (gc.id IS NULL) TotalAmount float64 // till_sales.total_amount — the funding this sale added } // sweepStaleRows resolves the stale pending rows of one table. Rows with a // square_payment_id are reconciled at Square first (COMPLETED → 'completed', // anything else → 'failed' exactly as the legacy bulk UPDATE did); rows // without one cannot be reconciled and are failed directly. A till_sales row // whose reconcile PROVES the charge never completed (NOT_FOUND / non-COMPLETED) // also claws back the funded gift card atomically with the failed mark; the // blind-fail path (no square_payment_id — the charge may have landed) never // claws back. Returns the total rows resolved and how many were rescued to // 'completed'. func sweepStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved int, completed int, err error) { switch table { case "payments", "till_sales": default: return 0, 0, fmt.Errorf("sweep: unknown stale table %q", table) } stale, err := fetchStaleRows(ctx, table, cutoff, false) if err != nil { return 0, 0, err } for _, r := range stale { if r.SquarePaymentID != "" { switch reconcileStalePaymentAtSquare(ctx, table, r.SquarePaymentID) { case staleReconcileCompleted: if table == "payments" && r.BookingID == nil { // Gift-card purchase row: the charge landed at Square but // the gift card was never delivered (C6). Completing the row // permanently blocks the same-key retry that delivers the // card — leave it pending and alert. leaveGiftCardPurchasePending(ctx, r) continue } if rescueStaleRowCompleted(ctx, table, r.ID) { resolved++ completed++ } continue case staleReconcileLeavePending: // Square's answer was ambiguous (transport/5xx) — the charge // may still be in flight at Square. Do NOT touch the row: the // next sweep run reconciles it again, and a same-key retry // must still be able to reuse the pending row if the charge // actually completed. log.Printf("Stale pending %s row %s left pending (Square reconcile ambiguous) — will retry next sweep", table, r.ID) continue } // staleReconcileDefinitivelyFailed falls through to the fail path. } // Fail path. A till-sale reconcile that PROVED the charge never // completed (NOT_FOUND / non-COMPLETED) claws back the funded gift // card atomically with the failed mark (HIGH-1). The blind-fail path // (no square_payment_id — the charge outcome is unknown) NEVER claws // back: the money may have landed at Square. if table == "till_sales" && r.SquarePaymentID != "" && r.HasGiftCard { if clawbackTillSaleFunding(ctx, r) { resolved++ } continue } if failStaleRow(ctx, table, r.ID) { resolved++ } } return resolved, completed, nil } // sweepKeyedStaleRows resolves stale pending rows that carry a stored // idempotency key but no square_payment_id — the lost-response case, where the // charge may have completed at Square with the response never reaching the // app. Such rows are reconciled at Square by replaying the key while it is // still inside Square's ~24h retention window (the earlier stalePendingKeyedAge // cutoff guarantees this): // // - a COMPLETED payment under the key → rescued to 'completed' with the real // square_payment_id written back (the customer WAS charged); // - a payment Square proves never happened (unknown key / FAILED status) → // marked 'failed', and a till sale's funded gift card is clawed back // (reconcile PROVED the funding has no charge behind it); // - an ambiguous answer (transport/5xx) → left pending for the next run; // - a row already older than the retention window when swept → cannot be // replayed trustworthily (an expired key would misread the probe rejection // as "never charged" even when the charge landed), so it is blind-failed // with a WARN exactly as the legacy sweep did. // // Returns the total rows resolved, how many were rescued to 'completed', and // how many were marked failed WITHOUT a reconcile proof (blind-fail WARNs). func sweepKeyedStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved, completed, unverifiable int, err error) { switch table { case "payments", "till_sales": default: return 0, 0, 0, fmt.Errorf("sweep: unknown stale table %q", table) } stale, err := fetchStaleRows(ctx, table, cutoff, true) if err != nil { return 0, 0, 0, err } // Square's idempotency-key retention window closes at stalePendingPaymentAge; // a row older than that when swept can no longer be replayed trustworthily. replayExpired := clock.Now().Add(-stalePendingPaymentAge) for _, r := range stale { if r.CreatedAt.Before(replayExpired) { // Key retention window already closed — replaying would misread an // expired key as "never charged". Blind-fail + WARN exactly as the // legacy sweep did; a till sale's funded gift card is NOT clawed // back (the charge outcome is unknown, the money may have landed). if failStaleRow(ctx, table, r.ID) { resolved++ unverifiable++ } log.Printf("Stale pending %s row %s has a stored idempotency key but is already past Square's key retention window — marked failed without a replay reconcile (may have been charged with a lost response)", table, r.ID) continue } switch res, sqPayID := reconcileStalePaymentByKey(ctx, table, r); res { case staleReconcileCompleted: if table == "payments" && r.BookingID == nil { // Gift-card purchase row (C6): the charge landed at Square but // the card was never delivered. Completing the row blocks the // same-key retry that delivers the card — leave it pending. leaveGiftCardPurchasePending(ctx, r) continue } if rescueKeyedStaleRowCompleted(ctx, table, r.ID, sqPayID) { resolved++ completed++ } case staleReconcileLeavePending: // Ambiguous replay — the charge may still be in flight. Leave the // row pending; the next sweep run reconciles it again. log.Printf("Stale pending %s row %s left pending (replay-by-key reconcile ambiguous) — will retry next sweep", table, r.ID) case staleReconcileDefinitivelyFailed: // Square proved the charge never happened (key unknown / FAILED) — // a till sale's funded gift card is clawed back atomically with the // failed mark, unlike the blind-fail path where the outcome is unknown. if table == "till_sales" && r.HasGiftCard { if clawbackTillSaleFunding(ctx, r) { resolved++ } continue } if failStaleRow(ctx, table, r.ID) { resolved++ } } } return resolved, completed, unverifiable, nil } // fetchStaleRows reads the stale pending rows of one table that are older than // cutoff. keyedOnly restricts the query to rows that can be reconciled by // replaying a stored idempotency key: those with a key but no square_payment_id // (rows WITH a square_payment_id are reconciled by payment id in the main // pass). The legacy till_sales sweep only touches card methods — cash and // on_the_house are committed synchronously and never pending, but the // predicate is kept so behaviour is byte-identical for any unexpected row. func fetchStaleRows(ctx context.Context, table string, cutoff time.Time, keyedOnly bool) ([]staleRow, error) { keyedPredicate := "" if keyedOnly { keyedPredicate = ` AND idempotency_key IS NOT NULL AND square_payment_id IS NULL` } methodFilter := "" if table == "till_sales" { methodFilter = ` AND payment_method IN ('online_square', 'in_person_card')` } var rows pgx.Rows var err error if table == "till_sales" { rows, err = db.Conn.Query(ctx, ` SELECT ts.id, COALESCE(ts.square_payment_id, ''), COALESCE(ts.idempotency_key, ''), COALESCE(ts.square_source_id, ''), COALESCE(ts.square_request_snapshot, ''), ts.created_at, ts.item_id, gc.redeemed_by, (ts.created_at = gc.created_at) AS is_create, (gc.id IS NOT NULL) AS has_gift_card, ts.total_amount FROM till_sales ts LEFT JOIN gift_cards gc ON gc.id = ts.item_id WHERE ts.status = 'pending' AND ts.created_at < $1`+methodFilter+keyedPredicate+` `, cutoff) } else { rows, err = db.Conn.Query(ctx, ` SELECT id, COALESCE(square_payment_id, ''), COALESCE(idempotency_key, ''), COALESCE(square_source_id, ''), COALESCE(square_request_snapshot, ''), created_at, amount, booking_id, created_by FROM `+table+` WHERE status = 'pending' AND created_at < $1`+keyedPredicate+` `, cutoff) } if err != nil { return nil, err } defer rows.Close() var stale []staleRow for rows.Next() { r, scanErr := scanStaleRow(table, rows) if scanErr != nil { log.Printf("Failed to scan stale pending row from %s: %v", table, scanErr) continue } stale = append(stale, r) } return stale, rows.Err() } // scanStaleRow scans one row produced by fetchStaleRows into a staleRow, // deriving the replay amount (in pence) from the stored pound figure — the // original CreatePayment amount the replay-by-key must repeat. func scanStaleRow(table string, rows pgx.Rows) (staleRow, error) { var r staleRow if table == "till_sales" { var itemID, redeemedBy, snapshot sql.NullString var isCreate *bool var hasGiftCard bool if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &itemID, &redeemedBy, &isCreate, &hasGiftCard, &r.TotalAmount); err != nil { return r, err } r.SquareRequestSnapshot = []byte(snapshot.String) r.ItemID = itemID.String if redeemedBy.Valid && redeemedBy.String != "" { r.RedeemToUserID = &redeemedBy.String } r.IsCreate = isCreate != nil && *isCreate r.HasGiftCard = hasGiftCard r.AmountPence = int64(math.Round(r.TotalAmount * 100)) return r, nil } var amount float64 var bookingID, createdBy, snapshot sql.NullString if err := rows.Scan(&r.ID, &r.SquarePaymentID, &r.IdempotencyKey, &r.SquareSourceID, &snapshot, &r.CreatedAt, &amount, &bookingID, &createdBy); err != nil { return r, err } r.SquareRequestSnapshot = []byte(snapshot.String) r.AmountPence = int64(math.Round(amount * 100)) if bookingID.Valid && bookingID.String != "" { b := bookingID.String r.BookingID = &b } if createdBy.Valid && createdBy.String != "" { c := createdBy.String r.CreatedBy = &c } return r, nil } // failStaleRow marks one stale pending row 'failed'. Returns true when the row // was updated (status was still 'pending'). func failStaleRow(ctx context.Context, table, id string) bool { tag, err := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'failed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, id) if err != nil { log.Printf("Failed to mark stale pending row %s failed: %v", id, err) return false } return int(tag.RowsAffected()) > 0 } // rescueStaleRowCompleted marks one stale pending row (whose square_payment_id // already resolves to a COMPLETED charge) 'completed'. Returns true when the // row was updated. func rescueStaleRowCompleted(ctx context.Context, table, id string) bool { tag, err := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'completed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, id) if err != nil { log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err) return false } return int(tag.RowsAffected()) > 0 } // rescueKeyedStaleRowCompleted marks a keyed stale pending row 'completed' with // the square_payment_id returned by the replay — the lost-response rescue. The // reconcile is deliberately minimal (status + square_payment_id + updated_at // only, no split/VAT recomputation): the row is >22h stale and this is a // reconciliation rescue, mirroring the F3 by-id rescue's minimality. Returns // true when the row was updated. func rescueKeyedStaleRowCompleted(ctx context.Context, table, id, squarePaymentID string) bool { tag, err := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'completed', square_payment_id = $1, updated_at = NOW() WHERE id = $2 AND status = 'pending' `, squarePaymentID, id) if err != nil { log.Printf("Failed to rescue stale pending row %s to completed: %v", id, err) return false } return int(tag.RowsAffected()) > 0 } // clawbackTillSaleFunding claws back a stale pending till sale's funded gift // card (atomically with the failed mark) after a reconcile PROVED the charge // never completed — the funding has no charge behind it. A sale with no gift // card is only marked failed. Returns true when the sale was resolved to // failed; false when it was already resolved by someone else, had no gift card, // or the clawback failed (CRITICAL logged). func clawbackTillSaleFunding(ctx context.Context, r staleRow) bool { action := "topup" if r.IsCreate { action = "create" } if revErr := revertGiftCardFunding(ctx, action, r.ItemID, r.TotalAmount, r.RedeemToUserID, r.ID); revErr != nil { if errors.Is(revErr, errTillSaleNotPending) { log.Printf("Stale pending till sale %s was already resolved (not pending) — skipping funding clawback", r.ID) return false } log.Printf("CRITICAL: failed to claw back gift card %s funding for stale till sale %s: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", r.ItemID, r.ID, revErr) return false } return true } // reconcileStalePaymentByKey asks Square for the authoritative status of the // charge made under a stale pending row's idempotency key and returns the // tri-state result. The replay sends an IDENTICAL body to the original charge: // the stored square_request_snapshot (the FULL request — source_id, key, // amount and every field the original carried). Replaying a partial body would // return IDEMPOTENCY_KEY_REUSED for a RETAINED key and strand the row pending // forever. Square's idempotency guarantee returns the ORIGINAL payment for a // retained key (never a second charge); a COMPLETED payment rescues the row to // 'completed' with its real payment id. A definitive 4xx rejection // (ErrReplayKeyNotRetained — Square attempted a real charge with the // expired/used source and refused it, so the charge never happened) or a // FAILED/CANCELED payment proves the charge never completed and fails the row. // IDEMPOTENCY_KEY_REUSED is NEVER proof of no charge: with an identical body it // can only mean the stored source differs from the original (a data bug), so // the row is left pending with a CRITICAL log. Any OTHER error (transport / // 5xx / ambiguous) leaves the row pending — the charge may still have completed // at Square. The second return value is the Square payment id of the completed // payment ("" otherwise), written back on a rescue. // // ENV CONTRACT (finding 4): the sweep and the charge process MUST run with the // SAME SQUARE_ENVIRONMENT and SQUARE_LOCATION_ID. The stored snapshot embeds // the location_id used at charge time; if the env/location changes between a // charge and its replay, the replayed wire body differs and a RETAINED key // returns IDEMPOTENCY_KEY_REUSED — stranding every retained-key row pending // (safe) until the 24h blind-fail. This is a single-location deployment; the // contract is enforced by ops (same env for the sweeper and the API), NOT by a // runtime equality check — deliberately comment-only. func reconcileStalePaymentByKey(ctx context.Context, table string, r staleRow) (staleReconcileResult, string) { snapshot := r.SquareRequestSnapshot // fallbackBody is true when the row has NO stored square_request_snapshot // and the minimal body (key + source + amount) is rebuilt. The rebuilt body // omits reference_id / customer_id / note / buyer_email_address that the // original charge carried, so for a RETAINED key real Square compares the // WHOLE body, sees the difference and returns IDEMPOTENCY_KEY_REUSED // (stranding the row pending) while the dev mock — which compares only the // source — returns the original payment (rescuing it). Both are money-safe; // the divergence is dev-vs-prod OBSERVABILITY only, and pre-launch there // are no prod legacy rows (every charge now stores its snapshot), so the // fallback semantics are deliberately left unchanged. fallbackBody := false if len(bytes.TrimSpace(snapshot)) == 0 { fallbackBody = true // Legacy row without a stored request snapshot — rebuild the minimal // identical body (key + source + amount) exactly as the pre-snapshot // replay did. Such rows can still be reconciled as long as the stored // source/amount match the original charge. fallback, mErr := json.Marshal(square.CreatePaymentReq{ Amount: r.AmountPence, Currency: "GBP", SourceID: r.SquareSourceID, IdempotencyKey: r.IdempotencyKey, }) if mErr != nil { log.Printf("Stale pending %s reconcile by key: failed to rebuild replay body for row %s (%v) — leaving pending", table, r.ID, mErr) return staleReconcileLeavePending, "" } snapshot = fallback } // The replay repeats the stored request snapshot verbatim so Square's // idempotency dedup returns the original payment for a retained key — which // requires the sweep to share SQUARE_ENVIRONMENT/SQUARE_LOCATION_ID with the // charge process (see the ENV CONTRACT above). pr, err := SquareClient.ReplayPaymentByKey(ctx, snapshot) if err != nil { if errors.Is(err, square.ErrReplayKeyNotRetained) { log.Printf("Stale pending %s reconcile by key: Square has no payment under the stored idempotency key (identical-body replay rejected) — marking failed; the charge provably never happened", table) return staleReconcileDefinitivelyFailed, "" } if square.ErrorCode(err) == "IDEMPOTENCY_KEY_REUSED" { if fallbackBody { // Snapshot-less fallback row: the minimal rebuilt body cannot be // identical to the original charge, so real Square rejects the // retained-key replay while the dev mock (source-aware only) // rescues it. The stranded row would otherwise be invisible // until the 24h blind-fail — surface it now (finding 2). log.Printf("CRITICAL: stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED via the minimal snapshot-less fallback body (key=%s) — real Square compares the WHOLE request (reference_id/customer_id/note/buyer_email_address absent from the rebuilt body) and rejects the identical-key replay, stranding the row pending; the dev mock would rescue it — dev-vs-prod observability divergence, NOT proof the charge never happened — MANUAL RECONCILIATION REQUIRED", table, r.IdempotencyKey) } else { log.Printf("CRITICAL: stale pending %s reconcile by key hit IDEMPOTENCY_KEY_REUSED — the stored square_source_id differs from the original charge's source (data bug); this is NOT proof the charge never happened — leaving pending — MANUAL RECONCILIATION REQUIRED", table) } return staleReconcileLeavePending, "" } log.Printf("Stale pending %s reconcile by idempotency key hit an ambiguous error (%v) — leaving pending for a later sweep run", table, err) return staleReconcileLeavePending, "" } // Square's terminal payment states are COMPLETED, CANCELED, FAILED; // APPROVED (authorization-only, delayed capture) and PENDING are // NON-terminal — both can still transition to COMPLETED, so clawing back // the funded gift card on either would risk reversing a charge that later // lands. This app always creates payments with autocomplete (default true), // so it never produces APPROVED/PENDING rows today, but the classification // must match Square's documented state machine. switch pr.Status { case "COMPLETED": return staleReconcileCompleted, pr.ID case "CANCELED", "FAILED": log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed, "" case "APPROVED", "PENDING": log.Printf("Stale pending %s is %q at Square (replay by key, non-terminal) — leaving pending for a later sweep run", table, pr.Status) return staleReconcileLeavePending, "" default: log.Printf("Stale pending %s is %q at Square (replay by key) — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed, "" } } // leaveGiftCardPurchasePending keeps a gift-card-purchase payment row (payments // table, booking_id NULL) pending after Square confirms the charge COMPLETED, // instead of rescuing it to 'completed'. Completing the row would permanently // block the same-key retry (BuyGiftCard) that reuses the pending record to // deliver the card — the customer would stay charged with no gift card (C6). // The row is left pending with a CRITICAL log + admin notification so the retry // can still deliver the card and an admin is alerted to reconcile manually. func leaveGiftCardPurchasePending(ctx context.Context, r staleRow) { log.Printf("CRITICAL: gift card purchase payment %s is COMPLETED at Square but the gift card was never issued (issue transaction failed, retry abandoned) — leaving the payment PENDING so a same-key retry can still deliver the card — MANUAL RECONCILIATION REQUIRED", r.ID) insertCriticalPaymentNotification(ctx, nil, r.CreatedBy) } // insertCriticalPaymentNotification surfaces an unresolved money event in the // admin notification centre (reason 'critical_payment_log' — the DB-backed // stand-in for the un-watched CRITICAL payment logs). bookingID is set when the // issue ties to a booking (untracked terminal charges); userID is set when it // ties to a user (gift-card purchases). The NOT EXISTS guard keeps ONE // notification per issue instead of one per sweep run. func insertCriticalPaymentNotification(ctx context.Context, bookingID, userID *string) { tag, err := db.Conn.Exec(ctx, ` INSERT INTO admin_notifications (reason, booking_id, user_id, created_at) SELECT 'critical_payment_log', $1, $2, NOW() WHERE NOT EXISTS ( SELECT 1 FROM admin_notifications an WHERE an.reason = 'critical_payment_log' AND an.booking_id IS NOT DISTINCT FROM $1 AND an.user_id IS NOT DISTINCT FROM $2 ) `, bookingID, userID) if err != nil { log.Printf("Failed to insert admin_notifications for critical payment issue (booking=%v user=%v): %v", bookingID, userID, err) return } if int(tag.RowsAffected()) > 0 { log.Printf("Inserted critical-payment admin notification (booking=%v user=%v)", bookingID, userID) } } // staleReconcileResult is the tri-state outcome of reconciling one stale // pending row against Square. Only a definitively-resolved outcome touches the // row: an ambiguous answer (transport error / 5xx) leaves it pending so a // same-key retry can still reuse it if the charge actually completed. type staleReconcileResult int const ( // staleReconcileLeavePending — Square's answer was ambiguous; the row stays // pending for the next sweep run. staleReconcileLeavePending staleReconcileResult = iota // staleReconcileCompleted — Square confirms the charge completed; rescue // the row to 'completed'. staleReconcileCompleted // staleReconcileDefinitivelyFailed — Square proves the charge never // completed (payment not found / non-completed status); mark the row // 'failed' exactly as the legacy bulk sweep did. staleReconcileDefinitivelyFailed ) // reconcileStalePaymentAtSquare asks Square for the authoritative status of a // stale pending charge and returns the tri-state result. A COMPLETED payment // rescues the row to 'completed'; a NOT_FOUND error or any non-COMPLETED status // proves the charge never completed and fails the row as the legacy bulk sweep // did. Any OTHER error (transport / 5xx / ambiguous) is NOT treated as a // definitive failure — the charge may still have completed at Square, and // marking the row failed would close the double-charge window (blocking a // same-key retry with a 409) even though the money moved. Such rows stay // pending for a later run. func reconcileStalePaymentAtSquare(ctx context.Context, table, squarePaymentID string) staleReconcileResult { pr, err := SquareClient.GetPayment(ctx, squarePaymentID) if err != nil { if squarePaymentErrorIsNotFound(err) { log.Printf("Stale pending %s reconcile: Square payment %s not found (%v) — marking failed as the legacy sweep would", table, squarePaymentID, err) return staleReconcileDefinitivelyFailed } log.Printf("Stale pending %s reconcile for Square payment %s hit an ambiguous error (%v) — leaving pending for a later sweep run", table, squarePaymentID, err) return staleReconcileLeavePending } // Square's terminal payment states are COMPLETED, CANCELED, FAILED; // APPROVED (authorization-only, delayed capture) and PENDING are // NON-terminal — both can still transition to COMPLETED (via // CompletePayment or delay_action=COMPLETE), so clawing back the funded // gift card on either would risk reversing a charge that later lands. // This app always creates payments with autocomplete (default true), so // it never produces APPROVED/PENDING rows today, but the classification // must match Square's documented state machine (librarian-verified // 2026-05-20). switch pr.Status { case "COMPLETED": return staleReconcileCompleted case "CANCELED", "FAILED": log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed case "APPROVED", "PENDING": log.Printf("Stale pending %s is %q at Square (non-terminal) — leaving pending for a later sweep run", table, pr.Status) return staleReconcileLeavePending default: log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed } } // squarePaymentErrorIsNotFound reports whether a GetPayment error proves the // payment does not exist at Square. The structured not-found check is primary // (square.IsNotFound: NOT_FOUND code / HTTP 404 status / plain "HTTP 404" body); // the message fallback covers only the dev mock's plain "payment not found" // error, which carries no structured code. func squarePaymentErrorIsNotFound(err error) bool { if err == nil { return false } if square.IsNotFound(err) { return true } // Any other structured Square error code is authoritative — never // substring-match its message. if square.ErrorCode(err) != "" { return false } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") } // squareHasCode reports whether err carries a structured Square error code // equal to any of codes. Errors without a structured code (the dev mock's // plain errors) match nothing. func squareHasCode(err error, codes ...string) bool { if err == nil { return false } code := square.ErrorCode(err) for _, c := range codes { if code == c { return true } } return false } // staleTerminalCheckoutAge is how old a still-pending terminal checkout must // be before the sweep cancels it. Terminal checkouts normally complete within // minutes; an hour is far past any legitimate card-reader interaction while // still short enough that a completed charge can never be misread as stale. const staleTerminalCheckoutAge = 1 * time.Hour // SweepStaleTerminalCheckouts cancels terminal (card-machine) checkouts that // are still PENDING/IN_PROGRESS long after they were created, so the terminal // stops waiting on a customer who walked away. A checkout created by // CreateTerminalPayment / CreateTillSale that is never polled would otherwise // sit live at Square indefinitely; if it later completes it is an invisible, // untracked charge. // // Two tables track live checkout IDs and are both swept: // - terminal_checkouts: booking terminal checkouts created by // CreateTerminalPayment. PENDING/IN_PROGRESS rows older than the cutoff // are resolved at Square first: a checkout still waiting at Square is // cancelled and re-checked once — if it completed during the cancel window // the row is marked 'completed' (the poll handler records the payment), // otherwise 'failed'; a COMPLETED checkout releases the in-flight guard // (the poll handler records the payment); a definitively // cancelled/expired checkout is marked 'failed'; an ambiguous status is // left for a later run. // - till_sales.square_checkout_id: card-machine till sales. A checkout still // waiting at Square is cancelled and the sale marked 'failed' (the // payment_status enum has no 'cancelled' value, and 'failed' is the same // terminal state the stale-pending sweep uses, blocking the till // pending-retry path); a checkout that completed during the cancel window // (or was already COMPLETED when first checked) is RECORDED by the sweep — // the sale is marked 'completed' with the returned square_payment_id, the // till mirror of the booking path's recordUntrackedTerminalPayment (a // never-polled sale would otherwise stay invisible in till reporting until // the 24h blind-fail). // // Each row is checked at Square FIRST and only cancelled when the checkout is // provably still waiting (ErrCheckoutPending): a COMPLETED checkout is never // cancelled, and a checkout whose status is unknown (transport error) is left // alone for a later run. After a cancel the checkout is re-checked once — the // customer may have completed the payment in the cancel window, in which case // the row is resolved to 'completed' rather than 'failed'. func SweepStaleTerminalCheckouts(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-staleTerminalCheckoutAge) var pending []staleTerminalCheckoutRow // Booking terminal checkouts (CreateTerminalPayment) live in the // terminal_checkouts table. A stale PENDING/IN_PROGRESS row means the // checkout is still live at Square (or was left after a crash / lost poll). rows, err := db.Conn.Query(ctx, ` SELECT 'terminal_checkout', checkout_id, checkout_id, booking_id FROM terminal_checkouts WHERE status IN ('PENDING', 'IN_PROGRESS') AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() // Till-sale card-machine checkouts are tracked on the till_sales row. rows, err = db.Conn.Query(ctx, ` SELECT 'till_sale', id, square_checkout_id, '' FROM till_sales WHERE status = 'pending' AND square_checkout_id IS NOT NULL AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID, &r.BookingID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() resolved := 0 for _, r := range pending { // A provisional (pre-Square) terminal_checkouts row carries a synthetic // "tmp-" checkout_id (or an empty one) — no checkout was ever created // at Square for it, so it is PROVABLY not live (R3). Resolve it to // failed directly without a Square round-trip; a hard crash between the // row insert and the Square CreateCheckout call is the only way one // exists. if r.CheckoutID == "" || strings.HasPrefix(r.CheckoutID, "tmp-") { if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Provisional (pre-Square) terminal checkout row %s (%s) resolved as failed — no live checkout at Square", r.RowID, r.Kind) continue } // Conservative status check: only cancel a checkout that is provably // still waiting at Square. A COMPLETED checkout must never be // cancelled, and an ambiguous status (network error) is left alone for // the next run. pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case errors.Is(gErr, square.ErrCheckoutPending): // Still live at the terminal — cancel it. A customer can complete // the payment in the small window between the GetCheckout above and // the cancel, so re-check once before marking the row failed: a // COMPLETED charge must never be recorded as failed. if cErr := SquareClient.CancelCheckout(ctx, r.CheckoutID); cErr != nil { log.Printf("Failed to cancel stale terminal checkout %s (%s %s): %v", r.CheckoutID, r.Kind, r.RowID, cErr) continue } recheck, rErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case rErr == nil && recheck.Status == "COMPLETED": // The customer completed the payment during the cancel window. // Record it if it was never polled/recorded — a COMPLETED // checkout must not stay an untracked charge (H4). Booking // checkouts record full payments rows; till-sale checkouts // record the sale row (finding 3). if r.Kind == "terminal_checkout" { if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, recheck) { resolved++ } } else if recordUntrackedTillSalePayment(ctx, r.RowID, recheck) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) but re-check shows COMPLETED — payment recorded by the sweep", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) case isTerminalCheckoutError(rErr) || errors.Is(rErr, square.ErrCheckoutPending): // The cancel landed (CANCELED / cancel-requested / expired / // still-reporting-pending-but-now-cancelled) — it can never // complete, so resolve the row to the terminal 'failed' state. // A till sale's funded gift card is clawed back (a cancel that // landed cannot later complete); a booking terminal_checkout // has no gift card. if r.Kind == "till_sale" { if clawBackTillSaleFunding(ctx, r.RowID) { resolved++ } } else if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) — marked failed", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) default: // The cancel succeeded but the re-check itself is ambiguous — // leave the row for a later run. log.Printf("Terminal checkout %s was cancelled but its re-check is ambiguous (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, rErr, r.Kind, r.RowID) } case gErr == nil && pr.Status == "COMPLETED": if r.Kind == "terminal_checkout" { // The checkout completed at Square but was never polled/recorded // (the frontend never called GetCheckoutStatus). Record the // payment rows now — otherwise the charge stays invisible to // refunds and TotalPaid (H4). if recordUntrackedTerminalPayment(ctx, r.CheckoutID, r.BookingID, pr) { resolved++ } } else if recordUntrackedTillSalePayment(ctx, r.RowID, pr) { // A never-polled COMPLETED till-sale checkout must not stay // pending until the 24h blind-fail (finding 3) — the charge is // real, so the sale is recorded completed with the payment id. resolved++ } case isTerminalCheckoutError(gErr): // The checkout is cancelled / cancel-requested / expired at Square. switch { case r.Kind == "till_sale" && isCheckoutDefinitivelyDead(gErr): // The error PROVES the checkout can never complete (CANCELED / // NOT_FOUND, and no bare CANCEL_REQUESTED) — claw back the // funded gift card along with the failed mark. if clawBackTillSaleFunding(ctx, r.RowID) { resolved++ } log.Printf("Terminal checkout %s is definitively terminal (%v) — marked till sale %s failed, gift-card funding clawed back", r.CheckoutID, gErr, r.RowID) case r.Kind == "till_sale": // CANCEL_REQUESTED-only: Square does not promise non-completion // in that state, so the charge may still land. Mark the sale // failed but DO NOT claw back the funded gift card. if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("CRITICAL: terminal checkout %s reports only CANCEL_REQUESTED (not provably dead) — till sale %s marked failed WITHOUT clawing back the funded gift card; verify at Square before re-issuing — MANUAL RECONCILIATION REQUIRED", r.CheckoutID, r.RowID) default: if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Terminal checkout %s is definitively terminal (%v) — marked %s %s failed", r.CheckoutID, gErr, r.Kind, r.RowID) } default: // Ambiguous transport/unknown status — leave for a later run. log.Printf("Terminal checkout %s status unknown (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, gErr, r.Kind, r.RowID) } } return resolved, nil } // staleTerminalCheckoutRow is one live-checkout row the sweep reads from // either table so it can resolve the checkout at Square before touching the // row. Kind is "terminal_checkout" (booking, terminal_checkouts table) or // "till_sale" (till_sales.square_checkout_id). BookingID is the booking the // terminal_checkouts row belongs to ("" for till sales) — needed to record an // untracked COMPLETED terminal charge (H4). type staleTerminalCheckoutRow struct { Kind string RowID string CheckoutID string BookingID string } // markTerminalCheckoutRowFailed moves one tracked row to the terminal 'failed' // state after its checkout is cancelled or proven terminal at Square. Returns // true when the row was updated (status was still active). func markTerminalCheckoutRowFailed(ctx context.Context, r staleTerminalCheckoutRow) bool { var table, where string if r.Kind == "till_sale" { table = "till_sales" where = "id = $1 AND status = 'pending'" } else { table = "terminal_checkouts" where = "checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')" } tag, err := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'failed', updated_at = NOW() WHERE `+where, r.RowID) if err != nil { log.Printf("Failed to mark %s %s failed: %v", r.Kind, r.RowID, err) return false } return int(tag.RowsAffected()) > 0 } // recordUntrackedTerminalPayment records the payments row(s) for a terminal // checkout that COMPLETED at Square but was never polled/recorded, then marks // the terminal_checkouts row COMPLETED. The stale sweep otherwise leaves a real // charge with NO payments row — invisible to refunds and TotalPaid (H4). It // reuses the exact insert convention of GetCheckoutStatus: the same advisory // lock key (serializes against a concurrent poll), the same dedup by // booking_id + square_payment_id, the same PaymentRecord shape and derived // idempotency key, and the same deposit/balance/tip split for a charge above // the remaining booking value. Returns true when the checkout row was resolved // (payment recorded or already recorded); false when recording failed (the // row is left pending so the next sweep re-runs the whole reconcile). func recordUntrackedTerminalPayment(ctx context.Context, checkoutID, bookingID string, pr *square.PaymentResult) bool { if pr == nil || pr.SquarePayID == "" { log.Printf("CRITICAL: terminal checkout %s is COMPLETED at Square but carries no Square payment ID — cannot record the payment — MANUAL RECONCILIATION REQUIRED", checkoutID) return false } service := NewPaymentService() // Serialize with the poll handler (GetCheckoutStatus): both record the same // Square payment, so the advisory lock + dedup SELECT prevent a double // insert (the idempotency_key UNIQUE constraint is the backstop). pinConn, err := db.Conn.Acquire(ctx) if err != nil { log.Printf("Failed to acquire connection for terminal-completion lock: %v", err) return false } defer pinConn.Release() lockOK, err := acquireAdvisoryLock(ctx, pinConn, "crussell:terminal:"+pr.SquarePayID) if err != nil { log.Printf("Failed to acquire terminal-completion serialization lock for %s: %v", pr.SquarePayID, err) return false } if !lockOK { log.Printf("Terminal-completion serialization lock for %s not acquired within bound — a poll is already recording this checkout", pr.SquarePayID) return false } defer func() { if _, err := pinConn.Exec(context.Background(), `SELECT pg_advisory_unlock(hashtext('crussell:terminal:' || $1))`, pr.SquarePayID); err != nil { log.Printf("Failed to release terminal-completion serialization lock for %s: %v", pr.SquarePayID, err) } }() tx, err := db.Conn.Begin(ctx) if err != nil { log.Printf("Failed to begin terminal-completion transaction: %v", err) return false } defer func() { if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { log.Printf("Failed to rollback terminal-completion transaction: %v", err) } }() // Dedup by Square payment ID: a concurrent poll (or a prior sweep run) // already recorded this charge — just release the in-flight guard. var existingID string if err := tx.QueryRow(ctx, ` SELECT id FROM payments WHERE booking_id = $1 AND square_payment_id = $2 `, bookingID, pr.SquarePayID).Scan(&existingID); err == nil { if _, upErr := tx.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, checkoutID); upErr != nil { log.Printf("Failed to mark terminal checkout %s completed: %v", checkoutID, upErr) return false } if cErr := tx.Commit(ctx); cErr != nil { log.Printf("Failed to commit terminal-completion transaction: %v", cErr) return false } return true } else if !errors.Is(err, pgx.ErrNoRows) { log.Printf("Failed to check for existing terminal payment: %v", err) return false } // The payment type the admin charged is recorded on the checkout row by // CreateTerminalPayment; fall back to 'full' for legacy rows. var checkoutPaymentType string if err := tx.QueryRow(ctx, ` SELECT payment_type FROM terminal_checkouts WHERE checkout_id = $1 `, checkoutID).Scan(&checkoutPaymentType); err != nil { if !errors.Is(err, pgx.ErrNoRows) { log.Printf("Failed to read payment type for checkout %s: %v", checkoutID, err) } checkoutPaymentType = "full" } idempotencyKey := bookingID + "-terminal-" + strconv.FormatInt(pr.Amount, 10) + "-" + pr.SquarePayID record := PaymentRecord{ BookingID: bookingID, PaymentType: checkoutPaymentType, PaymentMethod: "in_person_card", Status: "completed", Amount: float64(pr.Amount) / 100.0, SquarePaymentID: &pr.SquarePayID, IdempotencyKey: &idempotencyKey, Fees: float64(pr.Fees) / 100.0, CreatedAt: clock.Now(), UpdatedAt: clock.Now(), } // M4 split: a terminal charge above the remaining booking value is a tip — // record it as its own record so only the booking portion is refundable. var records []PaymentRecord bookingInfo, bErr := service.GetBookingPaymentInfo(ctx, bookingID) if bErr == nil && bookingInfo != nil { charged := float64(pr.Amount) / 100.0 remainingBookingValue := math.Max(0, bookingInfo.TotalAmount-bookingInfo.TotalPaid) bookingPortion := math.Min(charged, remainingBookingValue) bookingPortion = math.Round(bookingPortion*100) / 100 tipAmount := math.Round((charged-bookingPortion)*100) / 100 if tipAmount > 0.004 { records = buildTerminalSplitRecords(record, bookingInfo, bookingPortion, tipAmount) } } if len(records) == 0 { records = []PaymentRecord{record} } primary := records[0] paymentID, err := service.CreatePaymentRecordTx(ctx, tx, primary, nil) if err != nil { log.Printf("Failed to create payment record for untracked terminal charge %s: %v", pr.SquarePayID, err) return false } ApplyVATToBookingPayment(ctx, tx, paymentID) for _, rec := range records[1:] { pid, cErr := service.CreatePaymentRecordTx(ctx, tx, rec, nil) if cErr != nil { log.Printf("Failed to create terminal tip split record: %v", cErr) return false } ApplyVATToBookingPayment(ctx, tx, pid) } // Release the in-flight guard: this checkout is now recorded. if _, err := tx.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, checkoutID); err != nil { log.Printf("Failed to mark terminal checkout %s completed: %v", checkoutID, err) return false } if err := tx.Commit(ctx); err != nil { log.Printf("Failed to commit terminal-completion transaction: %v", err) return false } // The booking may now be fully paid — complete it like the poll handler does. completeFullyPaidBooking(ctx, bookingID) log.Printf("CRITICAL: recorded untracked terminal charge %s (booking %s) from stale checkout %s — payment row %s created (never polled by the frontend)", pr.SquarePayID, bookingID, checkoutID, paymentID) return true } // recordUntrackedTillSalePayment records a stale card-machine till sale whose // checkout COMPLETED at Square but was never polled/recorded: the sale is // marked 'completed' with the returned square_payment_id written back. A // never-polled COMPLETED checkout would otherwise leave the sale pending until // the 24h blind-fail — the charge is real but invisible in till reporting // (money-safe: the gift card was funded pre-charge, a blind-fail never claws // back, and failed rows reject retries — but the till reporting would be wrong, // finding 3). There is no split/deposit concept for till sales — a single // completed row, exactly the update the till poll handler (GetTillCheckoutStatus) // performs. The WHERE status = 'pending' guard makes it safe against a // concurrent poll: only one of them wins the update, the loser sees 0 rows and // backs off. Returns true when the sale was updated; false when the checkout // carries no Square payment id (the sale is left PENDING — never blind-failed // — with a CRITICAL log for manual reconciliation) or the sale was already // resolved by someone else. func recordUntrackedTillSalePayment(ctx context.Context, saleID string, pr *square.PaymentResult) bool { if pr == nil || pr.SquarePayID == "" { log.Printf("CRITICAL: terminal checkout is COMPLETED at Square but carries no Square payment ID — cannot record till sale %s — leaving it PENDING — MANUAL RECONCILIATION REQUIRED", saleID) return false } tag, err := db.Conn.Exec(ctx, ` UPDATE till_sales SET status = 'completed', square_payment_id = $1, updated_at = NOW() WHERE id = $2 AND status = 'pending' `, pr.SquarePayID, saleID) if err != nil { log.Printf("Failed to record untracked terminal till sale %s as completed: %v", saleID, err) return false } if int(tag.RowsAffected()) == 0 { // A concurrent poll (or a prior sweep run) already recorded the sale — // nothing left for this run to do. log.Printf("Terminal till sale %s was already resolved — skipping untracked terminal completion record", saleID) return false } log.Printf("CRITICAL: recorded untracked terminal till-sale charge %s (sale %s) from a stale checkout — sale marked completed (never polled by the frontend)", pr.SquarePayID, saleID) return true } // isTerminalCheckoutError reports whether a GetCheckout error proves the // checkout can never complete. Square's HTTP client returns ErrCheckoutPending // for a still-live checkout and surfaces a definitively CANCELED status as a // "square: checkout is CANCELED (not COMPLETED)" error; an expired // checkout returns a structured NOT_FOUND API error (the mock uses a plain // "checkout not found"). Any other error (timeout, 5xx) leaves the money state // ambiguous, so the checkout must stay in flight. // // Structured codes are authoritative when present: NOT_FOUND (via // square.IsNotFound) and an explicit CANCELED / CANCEL_REQUESTED code classify // as terminal. The formatted-message match applies only to errors that carry // no structured code — the dev mock's plain errors and the real client's // client-side "is (not COMPLETED)" error, which it synthesizes // without a squareAPIError. func isTerminalCheckoutError(err error) bool { if err == nil || errors.Is(err, square.ErrCheckoutPending) { return false } if square.IsNotFound(err) || squareHasCode(err, "CANCELED", "CANCEL_REQUESTED") { return true } // Any other structured Square error code is authoritative — never // substring-match its message. if square.ErrorCode(err) != "" { return false } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "CANCELED") || strings.Contains(msg, "CANCEL_REQUESTED") || strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") } // isCheckoutDefinitivelyDead reports whether a GetCheckout error PROVES the // checkout can never complete — the condition under which a till sale's funded // gift card may be clawed back. It is stricter than isTerminalCheckoutError: // a CANCEL_REQUESTED-only classification (Square does not promise non-completion // in that state) is NOT definitive proof, so the charge may still land and the // funding must stay put. Only an explicit CANCELED or NOT_FOUND status is // definitive; a CANCEL_REQUESTED message counts only when it ALSO carries // CANCELED. // // Structured codes are authoritative when present: CANCELED and NOT_FOUND are // definitive, a bare CANCEL_REQUESTED code is not. The formatted-message match // applies only to errors that carry no structured code (dev mock plain errors, // and the real client's client-side "is (not COMPLETED)" error). func isCheckoutDefinitivelyDead(err error) bool { if err == nil || errors.Is(err, square.ErrCheckoutPending) { return false } if code := square.ErrorCode(err); code != "" { return code == "CANCELED" || code == "NOT_FOUND" } // Non-structured errors: an HTTP 404 / plain 404 body is definitive // (IsNotFound); the message match covers the client-side CANCELED status // error and the mock's plain not-found errors. if square.IsNotFound(err) { return true } msg := strings.ToUpper(err.Error()) if !strings.Contains(msg, "CANCELED") && !strings.Contains(msg, "NOT_FOUND") && !strings.Contains(msg, "NOT FOUND") { return false } if strings.Contains(msg, "CANCEL_REQUESTED") && !strings.Contains(msg, "CANCELED") { return false } return true } // clawBackTillSaleFunding reverts the gift-card funding of a till sale whose // card-machine checkout is provably dead, marking the sale failed at the same // time. The terminal-checkout sweep's rows carry no gift-card context, so the // sale's item/amount/redeem target are looked up here (is_create via the // created_at equality with the gift card) and handed to the claim-first // revertGiftCardFunding. A sale with no gift card (future retail product / // orphaned item) is only marked failed. Returns true when the sale was // resolved to failed; false when it was already resolved by someone else, had // no gift card, or the clawback failed (CRITICAL logged). func clawBackTillSaleFunding(ctx context.Context, tillSaleID string) bool { var itemType string var itemID sql.NullString var totalAmount float64 var redeemedBy sql.NullString var isCreate *bool err := db.Conn.QueryRow(ctx, ` SELECT ts.item_type, ts.item_id, ts.total_amount, gc.redeemed_by, (ts.created_at = gc.created_at) AS is_create FROM till_sales ts LEFT JOIN gift_cards gc ON gc.id = ts.item_id WHERE ts.id = $1 `, tillSaleID).Scan(&itemType, &itemID, &totalAmount, &redeemedBy, &isCreate) if err != nil { log.Printf("CRITICAL: failed to look up till sale %s for funding clawback: %v — MANUAL RECONCILIATION REQUIRED", tillSaleID, err) return false } if itemType != "gift_card" || !itemID.Valid || itemID.String == "" || isCreate == nil { // No gift card to claw back — mark the sale failed without touching // any card. tag, upErr := db.Conn.Exec(ctx, ` UPDATE till_sales SET status = 'failed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, tillSaleID) if upErr != nil { log.Printf("Failed to mark till sale %s failed: %v", tillSaleID, upErr) return false } return int(tag.RowsAffected()) > 0 } action := "topup" if *isCreate { action = "create" } var redeem *string if redeemedBy.Valid && redeemedBy.String != "" { redeem = &redeemedBy.String } if revErr := revertGiftCardFunding(ctx, action, itemID.String, totalAmount, redeem, tillSaleID); revErr != nil { if errors.Is(revErr, errTillSaleNotPending) { log.Printf("Till sale %s was already resolved (not pending) — skipping funding clawback", tillSaleID) return false } log.Printf("CRITICAL: funding clawback for till sale %s (gift card %s) failed: %v — MANUAL RECONCILIATION REQUIRED: gift card may still be funded", tillSaleID, itemID.String, revErr) return false } return true }