//go:build test package webhooks // Round 9 adversarial regression tests — the webhook-vs-synchronous-path // double-completion race on the SAME booking/payment. // // The bug (R9): `reconcileCompletedPayments` runs the FULL completion // side-effects (split records, VAT, booking completion, loyalty/campaigns) // when a webhook promotes a pending payment row to completed. The synchronous // saved-card path (handlers.go CreateBookingPayment) ALSO runs its own // post-charge completion after Square returns. If the webhook arrives BETWEEN // Square returning the charge and the sync path's recheck transaction // committing, both paths contend on the same booking row, and the sync path — // whose completion UPDATE had NO `status='pending'` guard — would re-run its // side-effects on top of the webhook's: phantom duplicate split rows (whose // deterministic idempotency keys are UNIQUE) and a re-aligned primary row that // no longer reconciles to the Square charge. // // The coordinated fix has two halves: // - the sync path (handlers.go, owned by another agent) guards its own // completion flip on `status='pending'` and re-reads the payment status // before re-running side-effects; // - the webhook half (this suite): the webhook is ALREADY safe under that // fix because its reconcile SELECT matches only `status='pending'` rows // and its flip UPDATE is guarded on `status='pending'` — the payment // row's status is the single mutual-exclusion point, so whichever path // wins the flip, the side-effects run exactly once. // // These tests lock the webhook half: // 1. the webhook arrives AFTER the sync path committed its flip → the // pending-only SELECT matches nothing, the charge is acked as a known // settled replay, and NO side-effect re-runs (no duplicate split rows, no // re-completion of the booking); // 2. the webhook's SELECT reads the row while it is STILL pending (the sync // path's recheck tx holds the booking lock but has not committed), then // the sync path commits before the webhook's guarded flip runs → the // flip's `AND status='pending'` guard makes it a no-op and the webhook // never re-runs the side-effects. import ( "context" "encoding/json" "net/http" "net/http/httptest" "testing" "time" "crussell/db" ) // seedRound9SyncCompletedState simulates the synchronous saved-card path having // WON the race and fully completed the payment + booking: the primary row is // flipped to 'completed' and re-aligned to the deposit split, the balance split // is inserted with the deterministic idempotency key the split logic mints // (-split-1), and the fully-paid booking is completed. This is exactly the // ledger the webhook would produce if IT ran the side-effects, so any regression // that makes the webhook re-run them would collide on the UNIQUE idempotency // key or mint phantom rows. func seedRound9SyncCompletedState(t *testing.T, payID, bookingID, squarePaymentID, idemKey, userID string, total float64) { t.Helper() deposit := total * 0.5 // protected deposit max: 50% of total, pre-start split balance := total - deposit if _, err := db.Conn.Exec(context.Background(), `UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`, deposit, payID); err != nil { t.Fatalf("failed to flip payment to completed: %v", err) } if _, err := db.Conn.Exec(context.Background(), ` INSERT INTO payments (booking_id, payment_type, payment_method, amount, status, square_payment_id, idempotency_key, created_by, created_at, updated_at) VALUES ($1, 'balance', 'online_square', $2, 'completed', $3, $4, $5, NOW(), NOW()) `, bookingID, balance, squarePaymentID, idemKey+"-split-1", userID); err != nil { t.Fatalf("failed to seed balance split: %v", err) } if _, err := db.Conn.Exec(context.Background(), `UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil { t.Fatalf("failed to complete booking: %v", err) } } func countRound9BookingPayments(t *testing.T, bookingID string) int { t.Helper() var n int if err := db.Conn.QueryRow(context.Background(), "SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&n); err != nil { t.Fatalf("failed to count booking payments: %v", err) } return n } func countRound9ByIdempotencyKey(t *testing.T, idemKey string) int { t.Helper() var n int if err := db.Conn.QueryRow(context.Background(), "SELECT COUNT(*) FROM payments WHERE idempotency_key = $1", idemKey).Scan(&n); err != nil { t.Fatalf("failed to count payments by idempotency key: %v", err) } return n } // round9CompletedEvent builds a payment.updated webhook carrying a COMPLETED // Square charge. func round9CompletedEvent(squarePaymentID, idemKey string) SquareWebhookEvent { return SquareWebhookEvent{ Type: "payment.updated", EventID: "evt_round9_" + squarePaymentID, CreatedAt: nowInRFC3339(0), Data: json.RawMessage(`{ "type": "payment", "id": "` + squarePaymentID + `", "object": { "payment": { "id": "` + squarePaymentID + `", "status": "COMPLETED", "idempotency_key": "` + idemKey + `", "amount_money": {"amount": 5000, "currency": "GBP"} } } }`), } } // TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp locks fix half 1 (the R9 // race outcome where the SYNC path wins): the synchronous saved-card path has // already flipped the payment to 'completed', re-split it (deposit primary + // balance), and completed the booking BEFORE the webhook is delivered. The // webhook's reconcile SELECT matches only status='pending' rows, so it finds // nothing and acks the charge as a known settled replay — the row is never // re-flipped, no phantom split rows are minted, and the booking-completion // side-effects are not double-run. func TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp(t *testing.T) { const ( squarePaymentID = "sqp_round9_sync_first" idemKey = "round9-sync-key-1" total = 50.00 ) payID := createWebhookTestPayment(t, squarePaymentID, "pending") bookingID := attachWebhookTestBooking(t, payID, total) var userID string if err := db.Conn.QueryRow(context.Background(), "SELECT user_id FROM bookings WHERE id = $1", bookingID).Scan(&userID); err != nil { t.Fatalf("failed to read booking user: %v", err) } if _, err := db.Conn.Exec(context.Background(), "UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil { t.Fatalf("failed to set payment amount/key: %v", err) } // The sync path completed the charge and its booking first (the race // outcome this test locks). seedRound9SyncCompletedState(t, payID, bookingID, squarePaymentID, idemKey, userID, total) paymentsBefore := countRound9BookingPayments(t, bookingID) event := round9CompletedEvent(squarePaymentID, idemKey) w := deliverWebhook(t, event) if w.Code != http.StatusOK { t.Fatalf("expected 200 (known settled charge, plain replay), got %d: %s", w.Code, w.Body.String()) } // No double-completion: the payment stays exactly as the sync path left it. if got := getPaymentStatus(t, payID); got != "completed" { t.Errorf("expected payment to stay 'completed', got %q", got) } // The primary row keeps the sync path's split shape — never re-aligned or // re-split by the webhook. var primaryAmount float64 var primaryType string if err := db.Conn.QueryRow(context.Background(), "SELECT amount, payment_type FROM payments WHERE id = $1", payID).Scan(&primaryAmount, &primaryType); err != nil { t.Fatalf("failed to read primary payment: %v", err) } if primaryAmount != total/2 || primaryType != "deposit" { t.Errorf("expected primary to stay the deposit split (%.2f/deposit), got amount=%v type=%q", total/2, primaryAmount, primaryType) } // No phantom split rows: the ledger is byte-identical to the sync path's. if n := countRound9BookingPayments(t, bookingID); n != paymentsBefore { t.Errorf("expected the payment ledger unchanged (%d rows), got %d", paymentsBefore, n) } // No duplicate balance split: the deterministic key exists exactly once. if n := countRound9ByIdempotencyKey(t, idemKey+"-split-1"); n != 1 { t.Errorf("expected the balance split key to exist exactly once, got %d", n) } // Booking side-effects are not double-run: the booking stays 'completed' // (a second completion transition cannot fire). var bookingStatus string if err := db.Conn.QueryRow(context.Background(), "SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil { t.Fatalf("failed to read booking status: %v", err) } if bookingStatus != "completed" { t.Errorf("expected booking to stay 'completed', got %q", bookingStatus) } // The replay is acked and its dedup row committed. if n := countWebhookEvents(t, event.EventID); n != 1 { t.Errorf("expected 1 dedup row, got %d", n) } } // TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps locks the flip-guard // backstop — the IN-FLIGHT race (fix half 2). The webhook's pending-row SELECT // reads the row while it is STILL 'pending' (the sync path's recheck tx holds // the booking FOR UPDATE lock but has not committed its flip), then the sync // path commits (flip + booking completion) BEFORE the webhook's guarded flip // runs. The flip's `AND status='pending'` guard sees the committed 'completed' // row and becomes a no-op: the webhook never re-runs the split/completion // side-effects, so no duplicate rows and no double booking completion. func TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps(t *testing.T) { const ( squarePaymentID = "sqp_round9_sync_race" idemKey = "round9-race-key-1" total = 50.00 ) payID := createWebhookTestPayment(t, squarePaymentID, "pending") bookingID := attachWebhookTestBooking(t, payID, total) if _, err := db.Conn.Exec(context.Background(), "UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil { t.Fatalf("failed to set payment amount/key: %v", err) } // The sync path (postChargeRecheck) takes the booking FOR UPDATE lock and // holds it while its recheck tx is in flight. ctx := context.Background() syncTx, err := db.Conn.Begin(ctx) if err != nil { t.Fatalf("failed to begin sync-path tx: %v", err) } if _, err := syncTx.Exec(ctx, `SELECT 1 FROM bookings WHERE id = $1 FOR UPDATE`, bookingID); err != nil { syncTx.Rollback(ctx) t.Fatalf("failed to lock booking row: %v", err) } // Deliver the webhook in a goroutine. Signature/body are prepared on the // test goroutine (webhookTestEnv calls t.Setenv, which is test-goroutine // only); makeWebhookRequest itself touches no *testing.T. event := round9CompletedEvent(squarePaymentID, idemKey) body, err := json.Marshal(event) if err != nil { t.Fatalf("failed to marshal webhook event: %v", err) } sig := webhookTestEnv(t, body) done := make(chan *httptest.ResponseRecorder, 1) go func() { done <- makeWebhookRequest(body, sig, context.Background()) }() // Wait (bounded) until the webhook tx is actually blocked on the booking // FOR UPDATE lock — at that point its pending-row SELECT has ALREADY run and // seen the row as 'pending', so the flip-guard interleaving is exercised // deterministically. If it never blocks (a slow scheduler), the test still // holds: the commit below turns it into the no-op replay interleaving. waitDeadline := time.Now().Add(10 * time.Second) blocked := false for time.Now().Before(waitDeadline) { var n int if err := db.Conn.QueryRow(ctx, ` SELECT COUNT(*) FROM pg_stat_activity WHERE query ILIKE '%SELECT status FROM bookings WHERE id%' AND wait_event_type = 'Lock' AND pid <> pg_backend_pid() `).Scan(&n); err == nil && n > 0 { blocked = true break } time.Sleep(25 * time.Millisecond) } if !blocked { t.Logf("webhook never blocked on the booking lock — fell through to the no-op replay interleaving (assertions still hold)") } // The sync path commits: guarded flip + booking completion. if _, err := syncTx.Exec(ctx, `UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`, total/2, payID); err != nil { syncTx.Rollback(ctx) t.Fatalf("failed to flip payment in sync tx: %v", err) } if _, err := syncTx.Exec(ctx, `UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil { syncTx.Rollback(ctx) t.Fatalf("failed to complete booking in sync tx: %v", err) } if err := syncTx.Commit(ctx); err != nil { t.Fatalf("failed to commit sync-path tx: %v", err) } w := <-done if w.Code != http.StatusOK { t.Fatalf("expected 200 (guarded flip no-op), got %d: %s", w.Code, w.Body.String()) } // The payment was completed exactly once — by the sync path. if got := getPaymentStatus(t, payID); got != "completed" { t.Errorf("expected payment 'completed', got %q", got) } // No split rows beyond the primary: the webhook's side-effects never ran. if n := countRound9BookingPayments(t, bookingID); n != 1 { t.Errorf("expected exactly 1 payment row (the sync-flipped primary), got %d", n) } // The booking was completed by the sync path exactly once. var bookingStatus string if err := db.Conn.QueryRow(context.Background(), "SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil { t.Fatalf("failed to read booking status: %v", err) } if bookingStatus != "completed" { t.Errorf("expected booking 'completed' (the sync path completed it), got %q", bookingStatus) } if n := countWebhookEvents(t, event.EventID); n != 1 { t.Errorf("expected 1 dedup row, got %d", n) } }