popertots
726ac8cb65
Re-review (2 Oracle + security + QA + librarian + context-miner) surfaced fixes, all applied: (1) UserPaymentModal and the account-page gift-card buy now key the cached idempotency key on a stable 'new-card' sentinel instead of the cnon: nonce, so clearing the nonce on a failed charge no longer regenerates the key — a lost-response retry now dedups at Square instead of double-charging (the tip flows already keyed on amount only). (2) The create-with-redeem clawback now logs CRITICAL when the guarded balance reversal is blocked (previously silent), and its transaction DELETE is scoped to this sale instead of deleting every transaction on the card. (3) reconcileStalePaymentAtSquare now treats APPROVED/PENDING as non-terminal (leave pending) instead of definitively failed, matching Square's documented state machine. (4) GetTillCheckoutStatus returns 404 for a sale already swept to failed instead of reporting a live state. (5) The critical-payment scan job skips candidates whose booking was hard-deleted, so one orphan can no longer silence all critical alerts.
350 lines
11 KiB
Go
350 lines
11 KiB
Go
package jobs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"crussell/auth"
|
|
"crussell/db"
|
|
authHandlers "crussell/handlers/auth"
|
|
"crussell/handlers/payments"
|
|
"crussell/handlers/scheduling"
|
|
"crussell/handlers/user"
|
|
"crussell/mw"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// RegisterAll registers every background maintenance job on the scheduler.
|
|
// Call once during server startup, before s.Start().
|
|
func RegisterAll(s *Scheduler) {
|
|
// === HIGH FREQUENCY — every 5 minutes ===
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-reservations",
|
|
Schedule: "*/5 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupOldReservations,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-expired-deposits",
|
|
Schedule: "*/5 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredDeposits,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-rate-limiters",
|
|
Schedule: "*/5 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: mw.CleanupAllRateLimiters,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-gdpr-export-cache",
|
|
Schedule: "*/5 * * * *",
|
|
Timeout: 10 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: user.CleanupGDPRExportCache,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "sweep-pending-square-refunds",
|
|
Schedule: "*/5 * * * *",
|
|
Timeout: 60 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: payments.SweepPendingSquareRefunds,
|
|
})
|
|
|
|
// Offset from the refund sweep (which also writes payments rows) by one
|
|
// minute to avoid the two sweeps contending on the same table.
|
|
s.Register(Job{
|
|
Name: "sweep-stale-pending-payments",
|
|
Schedule: "1,6,11,16,21,26,31,36,41,46,51,56 * * * *",
|
|
Timeout: 60 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: payments.SweepStalePendingPayments,
|
|
})
|
|
|
|
// Cancels terminal (card-machine) checkouts still pending at Square after
|
|
// an hour — a never-polled checkout would otherwise sit live indefinitely
|
|
// and complete into an invisible, untracked charge.
|
|
s.Register(Job{
|
|
Name: "sweep-stale-terminal-checkouts",
|
|
Schedule: "*/15 * * * *",
|
|
Timeout: 60 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: payments.SweepStaleTerminalCheckouts,
|
|
})
|
|
|
|
// === MID FREQUENCY — every minute (progressive rate limiter was on 30s) ===
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-progressive-rate-limiter",
|
|
Schedule: "* * * * *",
|
|
Timeout: 10 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: mw.CleanupProgressiveRateLimiter,
|
|
})
|
|
|
|
// === MID FREQUENCY — hourly ===
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-expired-loyalty-redemptions",
|
|
Schedule: "0 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredLoyaltyRedemptions,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-old-idempotency-keys",
|
|
Schedule: "0 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupOldIdempotencyKeys,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-revoked-jtis",
|
|
Schedule: "0 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: auth.CleanupRevokedJTIs,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-stale-login-entries",
|
|
Schedule: "0 * * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: authHandlers.CleanupStaleLoginEntries,
|
|
})
|
|
|
|
// === LOW FREQUENCY — daily, off-peak (staggered to avoid DB contention) ===
|
|
|
|
s.Register(Job{
|
|
Name: "anonymize-stale-guest-accounts",
|
|
Schedule: "0 3 * * *",
|
|
Timeout: 5 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.AnonymizeStaleGuestAccounts,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-expired-financial-records",
|
|
Schedule: "0 4 * * *",
|
|
Timeout: 10 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredFinancialRecords,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-idle-accounts",
|
|
Schedule: "30 3 * * *",
|
|
Timeout: 5 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupIdleAccounts,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-expired-gift-cards",
|
|
Schedule: "0 5 * * *",
|
|
Timeout: 5 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredGiftCards,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-old-name-history",
|
|
Schedule: "30 4 * * *",
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupOldNameHistory,
|
|
})
|
|
|
|
// === STAGED HOURS CHANGE ===
|
|
|
|
s.Register(Job{
|
|
Name: "apply-default-hours",
|
|
Schedule: "5 0 * * *", // Daily at 00:05 — after midnight to avoid race
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.ApplyScheduledDefaultHours,
|
|
})
|
|
|
|
// === BUSINESS LOGIC JOBS ===
|
|
|
|
s.Register(Job{
|
|
Name: "notify-unpaid-1-week",
|
|
Schedule: "0 7 * * *", // Daily at 7am — end of business day + 7 days
|
|
Timeout: 2 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.NotifyUnpaidOneWeek,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "notify-unpaid-1-month",
|
|
Schedule: "30 7 * * *", // Daily at 7:30am (staggered from notify-unpaid-1-week)
|
|
Timeout: 2 * time.Minute,
|
|
Concurrency: 1,
|
|
Handler: scheduling.NotifyUnpaidOneMonth,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "transition-discount-campaigns",
|
|
Schedule: "0 * * * *", // Hourly
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.TransitionDiscountCampaigns,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-verification-codes",
|
|
Schedule: "0 2 * * *", // Daily at 2am
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredVerificationCodes,
|
|
})
|
|
|
|
s.Register(Job{
|
|
Name: "cleanup-refresh-tokens",
|
|
Schedule: "0 2 * * *", // Daily at 2am
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: scheduling.CleanupExpiredRefreshTokens,
|
|
})
|
|
|
|
// Staggered from cleanup-verification-codes / cleanup-refresh-tokens
|
|
// (both at 0 2 * * *) to avoid DB contention.
|
|
s.Register(Job{
|
|
Name: "sweep-square-webhook-events",
|
|
Schedule: "30 2 * * *", // Daily at 2:30am
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: SweepSquareWebhookEvents,
|
|
})
|
|
|
|
// Surfaces unresolved money events (stale pending payments/till sales,
|
|
// refunds at the retry cap) as admin notifications so the owner sees the
|
|
// "money may have moved at Square but the DB couldn't record it" situations
|
|
// that would otherwise live only in un-watched CRITICAL log lines (Gap
|
|
// Backlog T14). Retire this job when a proper log/alert pipeline (T14
|
|
// Sentry) lands — the notifications page is the stopgap while the app has
|
|
// none. Staggered at 2:45am between the 2am and 3am daily batches.
|
|
s.Register(Job{
|
|
Name: "scan-critical-payment-logs",
|
|
Schedule: "45 2 * * *", // Daily at 2:45am
|
|
Timeout: 30 * time.Second,
|
|
Concurrency: 1,
|
|
Handler: ScanCriticalPaymentLogs,
|
|
})
|
|
}
|
|
|
|
// SweepSquareWebhookEvents deletes square_webhook_events rows older than 90
|
|
// days. Every accepted webhook event_id is stored permanently for restart-safe
|
|
// dedup (payment.updated fires on every payment update), so without retention
|
|
// the table would grow without bound. 90 days comfortably exceeds Square's
|
|
// webhook replay window while keeping the table bounded.
|
|
func SweepSquareWebhookEvents(ctx context.Context) (int, error) {
|
|
tx, err := db.Conn.Begin(ctx)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := tx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
|
|
slog.Error("failed to rollback transaction", "err", err)
|
|
}
|
|
}()
|
|
|
|
tag, err := tx.Exec(ctx, `
|
|
DELETE FROM square_webhook_events
|
|
WHERE received_at < NOW() - INTERVAL '90 days'
|
|
`)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to sweep square webhook events: %w", err)
|
|
}
|
|
|
|
return int(tag.RowsAffected()), tx.Commit(ctx)
|
|
}
|
|
|
|
// criticalPaymentStaleAge is how long a pending payment / till sale may sit
|
|
// unresolved before the scan surfaces it to the admin notification centre. A
|
|
// pending row with no update in this window is exactly the "money may have
|
|
// moved at Square but the DB couldn't record it" situation the CRITICAL payment
|
|
// logs describe (handlers.go, giftcards.go, refunds.go, till.go). Deliberately
|
|
// shorter than the 24h stale-pending sweep (handlers/payments/sweep.go) so the
|
|
// owner hears about it while the row could still be rescued.
|
|
const criticalPaymentStaleAge = "2 hours"
|
|
|
|
// ScanCriticalPaymentLogs surfaces unresolved critical payment states as admin
|
|
// notifications (reason='critical_payment_log'), giving the owner an in-app
|
|
// view of money events that would otherwise be visible only in un-watched
|
|
// CRITICAL log lines (the app has no log/alert pipeline — Gap Backlog T14).
|
|
//
|
|
// Candidates:
|
|
// - payments / till_sales rows still 'pending' with no update for >2h: a
|
|
// lost-response charge the DB never recorded.
|
|
// - refunds rows still 'pending' at the 3-attempt retry cap: a refund the
|
|
// sweep could not resolve (money may have moved at Square).
|
|
//
|
|
// Each candidate inserts one admin_notifications row keyed on (reason,
|
|
// booking_id) — but ONLY if no unacknowledged notification for that key
|
|
// already exists (NULL-safe via IS NOT DISTINCT FROM, matching the
|
|
// insertRefundFailedNotifications dedup in handlers/payments/refunds.go), so
|
|
// the bell never floods. Acknowledging the notification re-arms the scan:
|
|
// while the row stays unresolved it is surfaced again on the next run.
|
|
//
|
|
// This is the "grep CRITICAL" the audit asked for, but DB-backed since the app
|
|
// has no log pipeline. When a proper log/alert pipeline (T14 Sentry) lands,
|
|
// this job can be retired.
|
|
func ScanCriticalPaymentLogs(ctx context.Context) (int, error) {
|
|
tag, err := db.Conn.Exec(ctx, `
|
|
INSERT INTO admin_notifications (reason, booking_id, created_at)
|
|
SELECT DISTINCT 'critical_payment_log'::admin_notification_reason, src.booking_id, NOW()
|
|
FROM (
|
|
-- Lost-response payments: still pending with no update past the threshold.
|
|
SELECT id, booking_id FROM payments
|
|
WHERE status = 'pending' AND updated_at < NOW() - INTERVAL '`+criticalPaymentStaleAge+`'
|
|
UNION ALL
|
|
-- Lost-response till sales (no booking linkage — booking_id NULL).
|
|
SELECT id, NULL::CHAR(12) FROM till_sales
|
|
WHERE status = 'pending' AND updated_at < NOW() - INTERVAL '`+criticalPaymentStaleAge+`'
|
|
UNION ALL
|
|
-- Refunds stuck at the 3-attempt retry cap.
|
|
SELECT id, booking_id FROM refunds
|
|
WHERE status = 'pending' AND refund_attempts >= 3
|
|
) src
|
|
WHERE NOT EXISTS (
|
|
SELECT 1 FROM admin_notifications an
|
|
WHERE an.reason = 'critical_payment_log'
|
|
AND an.booking_id IS NOT DISTINCT FROM src.booking_id
|
|
AND an.acknowledged_at IS NULL
|
|
)
|
|
-- admin_notifications.booking_id is a hard FK to bookings(id); a
|
|
-- candidate whose booking was hard-deleted (7-year retention) would
|
|
-- otherwise fail the whole INSERT and silently kill ALL critical
|
|
-- alerts. Skip orphans: keep NULL booking_ids (till sales) and only
|
|
-- payments/refunds whose booking still exists.
|
|
AND (src.booking_id IS NULL OR EXISTS (
|
|
SELECT 1 FROM bookings b WHERE b.id = src.booking_id
|
|
))
|
|
`)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to scan critical payment logs: %w", err)
|
|
}
|
|
n := int(tag.RowsAffected())
|
|
if n > 0 {
|
|
log.Printf("[SCAN] Inserted %d admin_notification(s) for unresolved critical payment states (pending payments/till sales > %s, refunds at retry cap)", n, criticalPaymentStaleAge)
|
|
}
|
|
return n, nil
|
|
}
|