test: round-9/10 adversarial suites — sync-path vs webhook completion, gift-card double-refund, sweep VAT, till lock, account lockout + erasure

- handlers_round9/round10: status-guarded flips, split-key hashing, cross-booking key 409, existingCount refund exclusion, SCA save-card exemption, routeNonCompletedPayment, no phantom split rows - giftcards_round10: saved-card SCA buy, cancel resume reconcile (pending blocks, diff-only re-issue, no over-refund) - sweep/till_round10: split-accurate VAT, all-tip VAT-free, status/key-changed skip, final-key lock held across charge - webhooks_round8/9: booking gate + M2, payable side-effects, unknown-event 503, refund-before-row 503, no double-complete after sync - account_round9: password lockout budgets, S3 erasure outbox, DAV in-tx deletion Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)
2026-08-22 00:34:51 +01:00
parent 985b114c8b
commit 1d6d3e2f8d
9 changed files with 2958 additions and 0 deletions
@@ -0,0 +1,309 @@
+//go:build test
+
+package webhooks
+
+// Round 9 adversarial regression tests — the webhook-vs-synchronous-path
+// double-completion race on the SAME booking/payment.
+//
+// The bug (R9): `reconcileCompletedPayments` runs the FULL completion
+// side-effects (split records, VAT, booking completion, loyalty/campaigns)
+// when a webhook promotes a pending payment row to completed. The synchronous
+// saved-card path (handlers.go CreateBookingPayment) ALSO runs its own
+// post-charge completion after Square returns. If the webhook arrives BETWEEN
+// Square returning the charge and the sync path's recheck transaction
+// committing, both paths contend on the same booking row, and the sync path —
+// whose completion UPDATE had NO `status='pending'` guard — would re-run its
+// side-effects on top of the webhook's: phantom duplicate split rows (whose
+// deterministic idempotency keys are UNIQUE) and a re-aligned primary row that
+// no longer reconciles to the Square charge.
+//
+// The coordinated fix has two halves:
+//   - the sync path (handlers.go, owned by another agent) guards its own
+//     completion flip on `status='pending'` and re-reads the payment status
+//     before re-running side-effects;
+//   - the webhook half (this suite): the webhook is ALREADY safe under that
+//     fix because its reconcile SELECT matches only `status='pending'` rows
+//     and its flip UPDATE is guarded on `status='pending'` — the payment
+//     row's status is the single mutual-exclusion point, so whichever path
+//     wins the flip, the side-effects run exactly once.
+//
+// These tests lock the webhook half:
+//  1. the webhook arrives AFTER the sync path committed its flip → the
+//     pending-only SELECT matches nothing, the charge is acked as a known
+//     settled replay, and NO side-effect re-runs (no duplicate split rows, no
+//     re-completion of the booking);
+//  2. the webhook's SELECT reads the row while it is STILL pending (the sync
+//     path's recheck tx holds the booking lock but has not committed), then
+//     the sync path commits before the webhook's guarded flip runs → the
+//     flip's `AND status='pending'` guard makes it a no-op and the webhook
+//     never re-runs the side-effects.
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"crussell/db"
+)
+
+// seedRound9SyncCompletedState simulates the synchronous saved-card path having
+// WON the race and fully completed the payment + booking: the primary row is
+// flipped to 'completed' and re-aligned to the deposit split, the balance split
+// is inserted with the deterministic idempotency key the split logic mints
+// (<key>-split-1), and the fully-paid booking is completed. This is exactly the
+// ledger the webhook would produce if IT ran the side-effects, so any regression
+// that makes the webhook re-run them would collide on the UNIQUE idempotency
+// key or mint phantom rows.
+func seedRound9SyncCompletedState(t *testing.T, payID, bookingID, squarePaymentID, idemKey, userID string, total float64) {
+	t.Helper()
+	deposit := total * 0.5 // protected deposit max: 50% of total, pre-start split
+	balance := total - deposit
+	if _, err := db.Conn.Exec(context.Background(),
+		`UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`,
+		deposit, payID); err != nil {
+		t.Fatalf("failed to flip payment to completed: %v", err)
+	}
+	if _, err := db.Conn.Exec(context.Background(), `
+		INSERT INTO payments (booking_id, payment_type, payment_method, amount, status, square_payment_id, idempotency_key, created_by, created_at, updated_at)
+		VALUES ($1, 'balance', 'online_square', $2, 'completed', $3, $4, $5, NOW(), NOW())
+	`, bookingID, balance, squarePaymentID, idemKey+"-split-1", userID); err != nil {
+		t.Fatalf("failed to seed balance split: %v", err)
+	}
+	if _, err := db.Conn.Exec(context.Background(),
+		`UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil {
+		t.Fatalf("failed to complete booking: %v", err)
+	}
+}
+
+func countRound9BookingPayments(t *testing.T, bookingID string) int {
+	t.Helper()
+	var n int
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&n); err != nil {
+		t.Fatalf("failed to count booking payments: %v", err)
+	}
+	return n
+}
+
+func countRound9ByIdempotencyKey(t *testing.T, idemKey string) int {
+	t.Helper()
+	var n int
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT COUNT(*) FROM payments WHERE idempotency_key = $1", idemKey).Scan(&n); err != nil {
+		t.Fatalf("failed to count payments by idempotency key: %v", err)
+	}
+	return n
+}
+
+// round9CompletedEvent builds a payment.updated webhook carrying a COMPLETED
+// Square charge.
+func round9CompletedEvent(squarePaymentID, idemKey string) SquareWebhookEvent {
+	return SquareWebhookEvent{
+		Type:      "payment.updated",
+		EventID:   "evt_round9_" + squarePaymentID,
+		CreatedAt: nowInRFC3339(0),
+		Data: json.RawMessage(`{
+			"type": "payment",
+			"id": "` + squarePaymentID + `",
+			"object": {
+				"payment": {
+					"id": "` + squarePaymentID + `",
+					"status": "COMPLETED",
+					"idempotency_key": "` + idemKey + `",
+					"amount_money": {"amount": 5000, "currency": "GBP"}
+				}
+			}
+		}`),
+	}
+}
+
+// TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp locks fix half 1 (the R9
+// race outcome where the SYNC path wins): the synchronous saved-card path has
+// already flipped the payment to 'completed', re-split it (deposit primary +
+// balance), and completed the booking BEFORE the webhook is delivered. The
+// webhook's reconcile SELECT matches only status='pending' rows, so it finds
+// nothing and acks the charge as a known settled replay — the row is never
+// re-flipped, no phantom split rows are minted, and the booking-completion
+// side-effects are not double-run.
+func TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp(t *testing.T) {
+	const (
+		squarePaymentID = "sqp_round9_sync_first"
+		idemKey         = "round9-sync-key-1"
+		total           = 50.00
+	)
+	payID := createWebhookTestPayment(t, squarePaymentID, "pending")
+	bookingID := attachWebhookTestBooking(t, payID, total)
+	var userID string
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT user_id FROM bookings WHERE id = $1", bookingID).Scan(&userID); err != nil {
+		t.Fatalf("failed to read booking user: %v", err)
+	}
+	if _, err := db.Conn.Exec(context.Background(),
+		"UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil {
+		t.Fatalf("failed to set payment amount/key: %v", err)
+	}
+	// The sync path completed the charge and its booking first (the race
+	// outcome this test locks).
+	seedRound9SyncCompletedState(t, payID, bookingID, squarePaymentID, idemKey, userID, total)
+	paymentsBefore := countRound9BookingPayments(t, bookingID)
+
+	event := round9CompletedEvent(squarePaymentID, idemKey)
+	w := deliverWebhook(t, event)
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200 (known settled charge, plain replay), got %d: %s", w.Code, w.Body.String())
+	}
+
+	// No double-completion: the payment stays exactly as the sync path left it.
+	if got := getPaymentStatus(t, payID); got != "completed" {
+		t.Errorf("expected payment to stay 'completed', got %q", got)
+	}
+	// The primary row keeps the sync path's split shape — never re-aligned or
+	// re-split by the webhook.
+	var primaryAmount float64
+	var primaryType string
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT amount, payment_type FROM payments WHERE id = $1", payID).Scan(&primaryAmount, &primaryType); err != nil {
+		t.Fatalf("failed to read primary payment: %v", err)
+	}
+	if primaryAmount != total/2 || primaryType != "deposit" {
+		t.Errorf("expected primary to stay the deposit split (%.2f/deposit), got amount=%v type=%q", total/2, primaryAmount, primaryType)
+	}
+	// No phantom split rows: the ledger is byte-identical to the sync path's.
+	if n := countRound9BookingPayments(t, bookingID); n != paymentsBefore {
+		t.Errorf("expected the payment ledger unchanged (%d rows), got %d", paymentsBefore, n)
+	}
+	// No duplicate balance split: the deterministic key exists exactly once.
+	if n := countRound9ByIdempotencyKey(t, idemKey+"-split-1"); n != 1 {
+		t.Errorf("expected the balance split key to exist exactly once, got %d", n)
+	}
+	// Booking side-effects are not double-run: the booking stays 'completed'
+	// (a second completion transition cannot fire).
+	var bookingStatus string
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil {
+		t.Fatalf("failed to read booking status: %v", err)
+	}
+	if bookingStatus != "completed" {
+		t.Errorf("expected booking to stay 'completed', got %q", bookingStatus)
+	}
+	// The replay is acked and its dedup row committed.
+	if n := countWebhookEvents(t, event.EventID); n != 1 {
+		t.Errorf("expected 1 dedup row, got %d", n)
+	}
+}
+
+// TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps locks the flip-guard
+// backstop — the IN-FLIGHT race (fix half 2). The webhook's pending-row SELECT
+// reads the row while it is STILL 'pending' (the sync path's recheck tx holds
+// the booking FOR UPDATE lock but has not committed its flip), then the sync
+// path commits (flip + booking completion) BEFORE the webhook's guarded flip
+// runs. The flip's `AND status='pending'` guard sees the committed 'completed'
+// row and becomes a no-op: the webhook never re-runs the split/completion
+// side-effects, so no duplicate rows and no double booking completion.
+func TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps(t *testing.T) {
+	const (
+		squarePaymentID = "sqp_round9_sync_race"
+		idemKey         = "round9-race-key-1"
+		total           = 50.00
+	)
+	payID := createWebhookTestPayment(t, squarePaymentID, "pending")
+	bookingID := attachWebhookTestBooking(t, payID, total)
+	if _, err := db.Conn.Exec(context.Background(),
+		"UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil {
+		t.Fatalf("failed to set payment amount/key: %v", err)
+	}
+
+	// The sync path (postChargeRecheck) takes the booking FOR UPDATE lock and
+	// holds it while its recheck tx is in flight.
+	ctx := context.Background()
+	syncTx, err := db.Conn.Begin(ctx)
+	if err != nil {
+		t.Fatalf("failed to begin sync-path tx: %v", err)
+	}
+	if _, err := syncTx.Exec(ctx, `SELECT 1 FROM bookings WHERE id = $1 FOR UPDATE`, bookingID); err != nil {
+		syncTx.Rollback(ctx)
+		t.Fatalf("failed to lock booking row: %v", err)
+	}
+
+	// Deliver the webhook in a goroutine. Signature/body are prepared on the
+	// test goroutine (webhookTestEnv calls t.Setenv, which is test-goroutine
+	// only); makeWebhookRequest itself touches no *testing.T.
+	event := round9CompletedEvent(squarePaymentID, idemKey)
+	body, err := json.Marshal(event)
+	if err != nil {
+		t.Fatalf("failed to marshal webhook event: %v", err)
+	}
+	sig := webhookTestEnv(t, body)
+	done := make(chan *httptest.ResponseRecorder, 1)
+	go func() {
+		done <- makeWebhookRequest(body, sig, context.Background())
+	}()
+
+	// Wait (bounded) until the webhook tx is actually blocked on the booking
+	// FOR UPDATE lock — at that point its pending-row SELECT has ALREADY run and
+	// seen the row as 'pending', so the flip-guard interleaving is exercised
+	// deterministically. If it never blocks (a slow scheduler), the test still
+	// holds: the commit below turns it into the no-op replay interleaving.
+	waitDeadline := time.Now().Add(10 * time.Second)
+	blocked := false
+	for time.Now().Before(waitDeadline) {
+		var n int
+		if err := db.Conn.QueryRow(ctx, `
+			SELECT COUNT(*) FROM pg_stat_activity
+			WHERE query ILIKE '%SELECT status FROM bookings WHERE id%'
+			  AND wait_event_type = 'Lock'
+			  AND pid <> pg_backend_pid()
+		`).Scan(&n); err == nil && n > 0 {
+			blocked = true
+			break
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	if !blocked {
+		t.Logf("webhook never blocked on the booking lock — fell through to the no-op replay interleaving (assertions still hold)")
+	}
+
+	// The sync path commits: guarded flip + booking completion.
+	if _, err := syncTx.Exec(ctx,
+		`UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`,
+		total/2, payID); err != nil {
+		syncTx.Rollback(ctx)
+		t.Fatalf("failed to flip payment in sync tx: %v", err)
+	}
+	if _, err := syncTx.Exec(ctx,
+		`UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil {
+		syncTx.Rollback(ctx)
+		t.Fatalf("failed to complete booking in sync tx: %v", err)
+	}
+	if err := syncTx.Commit(ctx); err != nil {
+		t.Fatalf("failed to commit sync-path tx: %v", err)
+	}
+
+	w := <-done
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200 (guarded flip no-op), got %d: %s", w.Code, w.Body.String())
+	}
+	// The payment was completed exactly once — by the sync path.
+	if got := getPaymentStatus(t, payID); got != "completed" {
+		t.Errorf("expected payment 'completed', got %q", got)
+	}
+	// No split rows beyond the primary: the webhook's side-effects never ran.
+	if n := countRound9BookingPayments(t, bookingID); n != 1 {
+		t.Errorf("expected exactly 1 payment row (the sync-flipped primary), got %d", n)
+	}
+	// The booking was completed by the sync path exactly once.
+	var bookingStatus string
+	if err := db.Conn.QueryRow(context.Background(),
+		"SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil {
+		t.Fatalf("failed to read booking status: %v", err)
+	}
+	if bookingStatus != "completed" {
+		t.Errorf("expected booking 'completed' (the sync path completed it), got %q", bookingStatus)
+	}
+	if n := countWebhookEvents(t, event.EventID); n != 1 {
+		t.Errorf("expected 1 dedup row, got %d", n)
+	}
+}