package payments import ( "context" "errors" "fmt" "log" "strings" "time" "crussell/clock" "crussell/db" "crussell/internal/square" ) // SweepStalePendingPayments resolves pending payment records that are older // than Square's idempotency-key retention window (~24h). A pending record // means the DB committed but the Square charge outcome is unknown; it // normally resolves on a same-key client retry. But if the client abandoned // the attempt, the record stays pending forever — and retrying it after the // key expires would ISSUE A SECOND CHARGE (Square no longer dedups). Failing // stale pendings closes that double-charge window: a late retry finds a // 'failed' record and stops instead of charging again. // // Before failing a row, the sweep reconciles it against Square: a // genuinely-charged row (Square success, DB post-charge failure) with a // square_payment_id is rescued to 'completed' instead of being swept to // 'failed' with no automatic resolution — the money would otherwise be lost // in limbo (MINOR-R3). Reconciliation is deliberately minimal: status + // updated_at only, no split/VAT recomputation (that is the handler's job; the // row is >24h stale and this is a reconciliation rescue). // // Only online/till card payments can be pending — cash/giftcard/on_the_house // are committed synchronously and never enter this state. Both the payments // table and till_sales carry pending card-sale rows and are swept here. const stalePendingPaymentAge = 24 * time.Hour func SweepStalePendingPayments(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-stalePendingPaymentAge) payCount, payCompleted, err := sweepStaleRows(ctx, "payments", cutoff) if err != nil { return 0, err } // till_sales rows for card payments (stored as 'online_square' or // 'in_person_card' in the payment_method enum — saved_card/online_square/ // card_machine requests all persist as one of those) can also be pending. // Sweep them too — a lost-response till sale would otherwise stay pending // and a retry after key retention would reuse the stored key → Square sees // an expired key → second charge (R3). Cash / on_the_house are committed // synchronously and never pending. tillCount, tillCompleted, err := sweepStaleRows(ctx, "till_sales", cutoff) if err != nil { return 0, err } total := payCount + tillCount if total > 0 { log.Printf("[SWEEP] Resolved %d stale pending payments (%d payments, %d till sales) older than %s — late retries will be rejected, preventing a second Square charge; %d reconciled to completed against Square (%d payments, %d till sales)", total, payCount, tillCount, stalePendingPaymentAge, payCompleted+tillCompleted, payCompleted, tillCompleted) } if payCount > 0 { log.Printf("CRITICAL: %d pending payments swept to failed may have been charged at Square with a lost response — manual reconciliation required before refunding/charging", payCount-payCompleted) } if tillCount > 0 { log.Printf("CRITICAL: %d pending till sales swept to failed may have been charged at Square with a lost response — manual reconciliation required", tillCount-tillCompleted) } return total, nil } // staleRow is one stale pending row read by the sweep so it can reconcile // rows that carry a Square reference BEFORE failing them. type staleRow struct { ID string SquarePaymentID string } // sweepStaleRows resolves the stale pending rows of one table. Rows with a // square_payment_id are reconciled at Square first (COMPLETED → 'completed', // anything else → 'failed' exactly as the legacy bulk UPDATE did); rows // without one cannot be reconciled and are failed directly. Returns the total // rows resolved and how many were rescued to 'completed'. func sweepStaleRows(ctx context.Context, table string, cutoff time.Time) (resolved int, completed int, err error) { switch table { case "payments", "till_sales": default: return 0, 0, fmt.Errorf("sweep: unknown stale table %q", table) } // The legacy till_sales sweep only touched card methods — cash and // on_the_house are committed synchronously and never pending, but keep the // predicate so behaviour is byte-identical for any unexpected row. methodFilter := "" if table == "till_sales" { methodFilter = ` AND payment_method IN ('online_square', 'in_person_card')` } rows, err := db.Conn.Query(ctx, ` SELECT id, COALESCE(square_payment_id, '') FROM `+table+` WHERE status = 'pending' AND created_at < $1`+methodFilter+` `, cutoff) if err != nil { return 0, 0, err } var stale []staleRow for rows.Next() { var r staleRow if err := rows.Scan(&r.ID, &r.SquarePaymentID); err != nil { log.Printf("Failed to scan stale pending row from %s: %v", table, err) continue } stale = append(stale, r) } rows.Close() for _, r := range stale { if r.SquarePaymentID != "" { switch reconcileStalePaymentAtSquare(ctx, table, r.SquarePaymentID) { case staleReconcileCompleted: if tag, upErr := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'completed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, r.ID); upErr != nil { log.Printf("Failed to rescue stale pending row %s to completed: %v", r.ID, upErr) } else if n := int(tag.RowsAffected()); n > 0 { resolved++ completed++ } continue case staleReconcileLeavePending: // Square's answer was ambiguous (transport/5xx) — the charge // may still be in flight at Square. Do NOT touch the row: the // next sweep run reconciles it again, and a same-key retry // must still be able to reuse the pending row if the charge // actually completed. log.Printf("Stale pending %s row %s left pending (Square reconcile ambiguous) — will retry next sweep", table, r.ID) continue } // staleReconcileDefinitivelyFailed falls through to the fail path. } if tag, upErr := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'failed', updated_at = NOW() WHERE id = $1 AND status = 'pending' `, r.ID); upErr != nil { log.Printf("Failed to mark stale pending row %s failed: %v", r.ID, upErr) } else if n := int(tag.RowsAffected()); n > 0 { resolved++ } } return resolved, completed, nil } // staleReconcileResult is the tri-state outcome of reconciling one stale // pending row against Square. Only a definitively-resolved outcome touches the // row: an ambiguous answer (transport error / 5xx) leaves it pending so a // same-key retry can still reuse it if the charge actually completed. type staleReconcileResult int const ( // staleReconcileLeavePending — Square's answer was ambiguous; the row stays // pending for the next sweep run. staleReconcileLeavePending staleReconcileResult = iota // staleReconcileCompleted — Square confirms the charge completed; rescue // the row to 'completed'. staleReconcileCompleted // staleReconcileDefinitivelyFailed — Square proves the charge never // completed (payment not found / non-completed status); mark the row // 'failed' exactly as the legacy bulk sweep did. staleReconcileDefinitivelyFailed ) // reconcileStalePaymentAtSquare asks Square for the authoritative status of a // stale pending charge and returns the tri-state result. A COMPLETED payment // rescues the row to 'completed'; a NOT_FOUND error or any non-COMPLETED status // proves the charge never completed and fails the row as the legacy bulk sweep // did. Any OTHER error (transport / 5xx / ambiguous) is NOT treated as a // definitive failure — the charge may still have completed at Square, and // marking the row failed would close the double-charge window (blocking a // same-key retry with a 409) even though the money moved. Such rows stay // pending for a later run. func reconcileStalePaymentAtSquare(ctx context.Context, table, squarePaymentID string) staleReconcileResult { pr, err := SquareClient.GetPayment(ctx, squarePaymentID) if err != nil { if squarePaymentErrorIsNotFound(err) { log.Printf("Stale pending %s reconcile: Square payment %s not found (%v) — marking failed as the legacy sweep would", table, squarePaymentID, err) return staleReconcileDefinitivelyFailed } log.Printf("Stale pending %s reconcile for Square payment %s hit an ambiguous error (%v) — leaving pending for a later sweep run", table, squarePaymentID, err) return staleReconcileLeavePending } if pr.Status != "COMPLETED" { log.Printf("Stale pending %s is %q at Square — marking failed", table, pr.Status) return staleReconcileDefinitivelyFailed } return staleReconcileCompleted } // squarePaymentErrorIsNotFound reports whether a GetPayment error proves the // payment does not exist at Square. The structured Square error code is the // primary check (square.ErrorCode); the message fallback also covers the dev // mock (a plain "payment not found" error) and a non-JSON 404 response. func squarePaymentErrorIsNotFound(err error) bool { if err == nil { return false } if square.ErrorCode(err) == "NOT_FOUND" { return true } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") || strings.Contains(msg, "HTTP 404") } // staleTerminalCheckoutAge is how old a still-pending terminal checkout must // be before the sweep cancels it. Terminal checkouts normally complete within // minutes; an hour is far past any legitimate card-reader interaction while // still short enough that a completed charge can never be misread as stale. const staleTerminalCheckoutAge = 1 * time.Hour // SweepStaleTerminalCheckouts cancels terminal (card-machine) checkouts that // are still PENDING/IN_PROGRESS long after they were created, so the terminal // stops waiting on a customer who walked away. A checkout created by // CreateTerminalPayment / CreateTillSale that is never polled would otherwise // sit live at Square indefinitely; if it later completes it is an invisible, // untracked charge. // // Two tables track live checkout IDs and are both swept: // - terminal_checkouts: booking terminal checkouts created by // CreateTerminalPayment. PENDING/IN_PROGRESS rows older than the cutoff // are resolved at Square first: a checkout still waiting at Square is // cancelled and re-checked once — if it completed during the cancel window // the row is marked 'completed' (the poll handler records the payment), // otherwise 'failed'; a COMPLETED checkout releases the in-flight guard // (the poll handler records the payment); a definitively // cancelled/expired checkout is marked 'failed'; an ambiguous status is // left for a later run. // - till_sales.square_checkout_id: card-machine till sales. A checkout still // waiting at Square is cancelled and the sale marked 'failed' (the // payment_status enum has no 'cancelled' value, and 'failed' is the same // terminal state the stale-pending sweep uses, blocking the till // pending-retry path); a checkout that completed during the cancel window // leaves the sale pending for the poll handler to record. // // Each row is checked at Square FIRST and only cancelled when the checkout is // provably still waiting (ErrCheckoutPending): a COMPLETED checkout is never // cancelled, and a checkout whose status is unknown (transport error) is left // alone for a later run. After a cancel the checkout is re-checked once — the // customer may have completed the payment in the cancel window, in which case // the row is resolved to 'completed' rather than 'failed'. func SweepStaleTerminalCheckouts(ctx context.Context) (int, error) { cutoff := clock.Now().Add(-staleTerminalCheckoutAge) var pending []staleTerminalCheckoutRow // Booking terminal checkouts (CreateTerminalPayment) live in the // terminal_checkouts table. A stale PENDING/IN_PROGRESS row means the // checkout is still live at Square (or was left after a crash / lost poll). rows, err := db.Conn.Query(ctx, ` SELECT 'terminal_checkout', checkout_id, checkout_id FROM terminal_checkouts WHERE status IN ('PENDING', 'IN_PROGRESS') AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() // Till-sale card-machine checkouts are tracked on the till_sales row. rows, err = db.Conn.Query(ctx, ` SELECT 'till_sale', id, square_checkout_id FROM till_sales WHERE status = 'pending' AND square_checkout_id IS NOT NULL AND created_at < $1 `, cutoff) if err != nil { return 0, err } for rows.Next() { var r staleTerminalCheckoutRow if err := rows.Scan(&r.Kind, &r.RowID, &r.CheckoutID); err != nil { log.Printf("Failed to scan stale terminal checkout row: %v", err) continue } pending = append(pending, r) } rows.Close() resolved := 0 for _, r := range pending { // A provisional (pre-Square) terminal_checkouts row carries a synthetic // "tmp-" checkout_id (or an empty one) — no checkout was ever created // at Square for it, so it is PROVABLY not live (R3). Resolve it to // failed directly without a Square round-trip; a hard crash between the // row insert and the Square CreateCheckout call is the only way one // exists. if r.CheckoutID == "" || strings.HasPrefix(r.CheckoutID, "tmp-") { if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Provisional (pre-Square) terminal checkout row %s (%s) resolved as failed — no live checkout at Square", r.RowID, r.Kind) continue } // Conservative status check: only cancel a checkout that is provably // still waiting at Square. A COMPLETED checkout must never be // cancelled, and an ambiguous status (network error) is left alone for // the next run. pr, gErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case errors.Is(gErr, square.ErrCheckoutPending): // Still live at the terminal — cancel it. A customer can complete // the payment in the small window between the GetCheckout above and // the cancel, so re-check once before marking the row failed: a // COMPLETED charge must never be recorded as failed. if cErr := SquareClient.CancelCheckout(ctx, r.CheckoutID); cErr != nil { log.Printf("Failed to cancel stale terminal checkout %s (%s %s): %v", r.CheckoutID, r.Kind, r.RowID, cErr) continue } recheck, rErr := SquareClient.GetCheckout(ctx, r.CheckoutID) switch { case rErr == nil && recheck.Status == "COMPLETED": // The customer completed the payment during the cancel window. // The poll handler records it — mark the row COMPLETED (or // leave the till sale pending) instead of failed. if r.Kind == "terminal_checkout" { if tag, upErr := db.Conn.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, r.RowID); upErr != nil { log.Printf("Failed to mark terminal checkout %s completed after cancel re-check: %v", r.RowID, upErr) } else if int(tag.RowsAffected()) > 0 { resolved++ } } else { log.Printf("Terminal checkout %s completed during sweep cancel — leaving sale %s pending (poll handler records it)", r.CheckoutID, r.RowID) } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) but re-check shows COMPLETED — recorded as completed, payment handled by the poll handler", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) case isTerminalCheckoutError(rErr) || errors.Is(rErr, square.ErrCheckoutPending): // The cancel landed (CANCELED / cancel-requested / expired / // still-reporting-pending-but-now-cancelled) — it can never // complete, so resolve the row to the terminal 'failed' state. if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Cancelled stale terminal checkout %s (%s %s, pending >%s) — marked failed", r.CheckoutID, r.Kind, r.RowID, staleTerminalCheckoutAge) default: // The cancel succeeded but the re-check itself is ambiguous — // leave the row for a later run. log.Printf("Terminal checkout %s was cancelled but its re-check is ambiguous (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, rErr, r.Kind, r.RowID) } case gErr == nil && pr.Status == "COMPLETED": if r.Kind == "terminal_checkout" { // The payment is recorded by the poll handler; release the // in-flight guard so a fresh charge can be created. if tag, upErr := db.Conn.Exec(ctx, ` UPDATE terminal_checkouts SET status = 'COMPLETED', updated_at = NOW() WHERE checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS') `, r.RowID); upErr != nil { log.Printf("Failed to mark terminal checkout %s completed: %v", r.RowID, upErr) } else if int(tag.RowsAffected()) > 0 { resolved++ } } else { // The till poll handler records it — leave the sale pending. log.Printf("Terminal checkout %s already COMPLETED at Square — leaving sale %s pending (poll handler records it)", r.CheckoutID, r.RowID) } case isTerminalCheckoutError(gErr): // Definitely cancelled / cancel-requested / expired — the checkout // can never complete, so resolve it to the terminal 'failed' state. if markTerminalCheckoutRowFailed(ctx, r) { resolved++ } log.Printf("Terminal checkout %s is definitively terminal (%v) — marked %s %s failed", r.CheckoutID, gErr, r.Kind, r.RowID) default: // Ambiguous transport/unknown status — leave for a later run. log.Printf("Terminal checkout %s status unknown (%v) — leaving %s %s pending for a later sweep", r.CheckoutID, gErr, r.Kind, r.RowID) } } return resolved, nil } // staleTerminalCheckoutRow is one live-checkout row the sweep reads from // either table so it can resolve the checkout at Square before touching the // row. Kind is "terminal_checkout" (booking, terminal_checkouts table) or // "till_sale" (till_sales.square_checkout_id). type staleTerminalCheckoutRow struct { Kind string RowID string CheckoutID string } // markTerminalCheckoutRowFailed moves one tracked row to the terminal 'failed' // state after its checkout is cancelled or proven terminal at Square. Returns // true when the row was updated (status was still active). func markTerminalCheckoutRowFailed(ctx context.Context, r staleTerminalCheckoutRow) bool { var table, where string if r.Kind == "till_sale" { table = "till_sales" where = "id = $1 AND status = 'pending'" } else { table = "terminal_checkouts" where = "checkout_id = $1 AND status IN ('PENDING', 'IN_PROGRESS')" } tag, err := db.Conn.Exec(ctx, ` UPDATE `+table+` SET status = 'failed', updated_at = NOW() WHERE `+where, r.RowID) if err != nil { log.Printf("Failed to mark %s %s failed: %v", r.Kind, r.RowID, err) return false } return int(tag.RowsAffected()) > 0 } // isTerminalCheckoutError reports whether a GetCheckout error proves the // checkout can never complete. Square's HTTP client returns ErrCheckoutPending // for a still-live checkout and surfaces a definitively CANCELED status as a // "square: checkout is CANCELED (not COMPLETED)" error; an expired // checkout returns a NOT_FOUND API error (the mock uses "checkout not found"). // Any other error (timeout, 5xx) leaves the money state ambiguous, so the // checkout must stay in flight. func isTerminalCheckoutError(err error) bool { if err == nil || errors.Is(err, square.ErrCheckoutPending) { return false } msg := strings.ToUpper(err.Error()) return strings.Contains(msg, "CANCELED") || strings.Contains(msg, "CANCEL_REQUESTED") || strings.Contains(msg, "NOT_FOUND") || strings.Contains(msg, "NOT FOUND") }