- handlers_round9/round10: status-guarded flips, split-key hashing, cross-booking key 409, existingCount refund exclusion, SCA save-card exemption, routeNonCompletedPayment, no phantom split rows - giftcards_round10: saved-card SCA buy, cancel resume reconcile (pending blocks, diff-only re-issue, no over-refund) - sweep/till_round10: split-accurate VAT, all-tip VAT-free, status/key-changed skip, final-key lock held across charge - webhooks_round8/9: booking gate + M2, payable side-effects, unknown-event 503, refund-before-row 503, no double-complete after sync - account_round9: password lockout budgets, S3 erasure outbox, DAV in-tx deletion Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)
310 lines
14 KiB
Go
310 lines
14 KiB
Go
//go:build test
|
|
|
|
package webhooks
|
|
|
|
// Round 9 adversarial regression tests — the webhook-vs-synchronous-path
|
|
// double-completion race on the SAME booking/payment.
|
|
//
|
|
// The bug (R9): `reconcileCompletedPayments` runs the FULL completion
|
|
// side-effects (split records, VAT, booking completion, loyalty/campaigns)
|
|
// when a webhook promotes a pending payment row to completed. The synchronous
|
|
// saved-card path (handlers.go CreateBookingPayment) ALSO runs its own
|
|
// post-charge completion after Square returns. If the webhook arrives BETWEEN
|
|
// Square returning the charge and the sync path's recheck transaction
|
|
// committing, both paths contend on the same booking row, and the sync path —
|
|
// whose completion UPDATE had NO `status='pending'` guard — would re-run its
|
|
// side-effects on top of the webhook's: phantom duplicate split rows (whose
|
|
// deterministic idempotency keys are UNIQUE) and a re-aligned primary row that
|
|
// no longer reconciles to the Square charge.
|
|
//
|
|
// The coordinated fix has two halves:
|
|
// - the sync path (handlers.go, owned by another agent) guards its own
|
|
// completion flip on `status='pending'` and re-reads the payment status
|
|
// before re-running side-effects;
|
|
// - the webhook half (this suite): the webhook is ALREADY safe under that
|
|
// fix because its reconcile SELECT matches only `status='pending'` rows
|
|
// and its flip UPDATE is guarded on `status='pending'` — the payment
|
|
// row's status is the single mutual-exclusion point, so whichever path
|
|
// wins the flip, the side-effects run exactly once.
|
|
//
|
|
// These tests lock the webhook half:
|
|
// 1. the webhook arrives AFTER the sync path committed its flip → the
|
|
// pending-only SELECT matches nothing, the charge is acked as a known
|
|
// settled replay, and NO side-effect re-runs (no duplicate split rows, no
|
|
// re-completion of the booking);
|
|
// 2. the webhook's SELECT reads the row while it is STILL pending (the sync
|
|
// path's recheck tx holds the booking lock but has not committed), then
|
|
// the sync path commits before the webhook's guarded flip runs → the
|
|
// flip's `AND status='pending'` guard makes it a no-op and the webhook
|
|
// never re-runs the side-effects.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"crussell/db"
|
|
)
|
|
|
|
// seedRound9SyncCompletedState simulates the synchronous saved-card path having
|
|
// WON the race and fully completed the payment + booking: the primary row is
|
|
// flipped to 'completed' and re-aligned to the deposit split, the balance split
|
|
// is inserted with the deterministic idempotency key the split logic mints
|
|
// (<key>-split-1), and the fully-paid booking is completed. This is exactly the
|
|
// ledger the webhook would produce if IT ran the side-effects, so any regression
|
|
// that makes the webhook re-run them would collide on the UNIQUE idempotency
|
|
// key or mint phantom rows.
|
|
func seedRound9SyncCompletedState(t *testing.T, payID, bookingID, squarePaymentID, idemKey, userID string, total float64) {
|
|
t.Helper()
|
|
deposit := total * 0.5 // protected deposit max: 50% of total, pre-start split
|
|
balance := total - deposit
|
|
if _, err := db.Conn.Exec(context.Background(),
|
|
`UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`,
|
|
deposit, payID); err != nil {
|
|
t.Fatalf("failed to flip payment to completed: %v", err)
|
|
}
|
|
if _, err := db.Conn.Exec(context.Background(), `
|
|
INSERT INTO payments (booking_id, payment_type, payment_method, amount, status, square_payment_id, idempotency_key, created_by, created_at, updated_at)
|
|
VALUES ($1, 'balance', 'online_square', $2, 'completed', $3, $4, $5, NOW(), NOW())
|
|
`, bookingID, balance, squarePaymentID, idemKey+"-split-1", userID); err != nil {
|
|
t.Fatalf("failed to seed balance split: %v", err)
|
|
}
|
|
if _, err := db.Conn.Exec(context.Background(),
|
|
`UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil {
|
|
t.Fatalf("failed to complete booking: %v", err)
|
|
}
|
|
}
|
|
|
|
func countRound9BookingPayments(t *testing.T, bookingID string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&n); err != nil {
|
|
t.Fatalf("failed to count booking payments: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
func countRound9ByIdempotencyKey(t *testing.T, idemKey string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT COUNT(*) FROM payments WHERE idempotency_key = $1", idemKey).Scan(&n); err != nil {
|
|
t.Fatalf("failed to count payments by idempotency key: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// round9CompletedEvent builds a payment.updated webhook carrying a COMPLETED
|
|
// Square charge.
|
|
func round9CompletedEvent(squarePaymentID, idemKey string) SquareWebhookEvent {
|
|
return SquareWebhookEvent{
|
|
Type: "payment.updated",
|
|
EventID: "evt_round9_" + squarePaymentID,
|
|
CreatedAt: nowInRFC3339(0),
|
|
Data: json.RawMessage(`{
|
|
"type": "payment",
|
|
"id": "` + squarePaymentID + `",
|
|
"object": {
|
|
"payment": {
|
|
"id": "` + squarePaymentID + `",
|
|
"status": "COMPLETED",
|
|
"idempotency_key": "` + idemKey + `",
|
|
"amount_money": {"amount": 5000, "currency": "GBP"}
|
|
}
|
|
}
|
|
}`),
|
|
}
|
|
}
|
|
|
|
// TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp locks fix half 1 (the R9
|
|
// race outcome where the SYNC path wins): the synchronous saved-card path has
|
|
// already flipped the payment to 'completed', re-split it (deposit primary +
|
|
// balance), and completed the booking BEFORE the webhook is delivered. The
|
|
// webhook's reconcile SELECT matches only status='pending' rows, so it finds
|
|
// nothing and acks the charge as a known settled replay — the row is never
|
|
// re-flipped, no phantom split rows are minted, and the booking-completion
|
|
// side-effects are not double-run.
|
|
func TestWebhook_Round9_WebhookAfterSyncCompletion_NoOp(t *testing.T) {
|
|
const (
|
|
squarePaymentID = "sqp_round9_sync_first"
|
|
idemKey = "round9-sync-key-1"
|
|
total = 50.00
|
|
)
|
|
payID := createWebhookTestPayment(t, squarePaymentID, "pending")
|
|
bookingID := attachWebhookTestBooking(t, payID, total)
|
|
var userID string
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT user_id FROM bookings WHERE id = $1", bookingID).Scan(&userID); err != nil {
|
|
t.Fatalf("failed to read booking user: %v", err)
|
|
}
|
|
if _, err := db.Conn.Exec(context.Background(),
|
|
"UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil {
|
|
t.Fatalf("failed to set payment amount/key: %v", err)
|
|
}
|
|
// The sync path completed the charge and its booking first (the race
|
|
// outcome this test locks).
|
|
seedRound9SyncCompletedState(t, payID, bookingID, squarePaymentID, idemKey, userID, total)
|
|
paymentsBefore := countRound9BookingPayments(t, bookingID)
|
|
|
|
event := round9CompletedEvent(squarePaymentID, idemKey)
|
|
w := deliverWebhook(t, event)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 (known settled charge, plain replay), got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// No double-completion: the payment stays exactly as the sync path left it.
|
|
if got := getPaymentStatus(t, payID); got != "completed" {
|
|
t.Errorf("expected payment to stay 'completed', got %q", got)
|
|
}
|
|
// The primary row keeps the sync path's split shape — never re-aligned or
|
|
// re-split by the webhook.
|
|
var primaryAmount float64
|
|
var primaryType string
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT amount, payment_type FROM payments WHERE id = $1", payID).Scan(&primaryAmount, &primaryType); err != nil {
|
|
t.Fatalf("failed to read primary payment: %v", err)
|
|
}
|
|
if primaryAmount != total/2 || primaryType != "deposit" {
|
|
t.Errorf("expected primary to stay the deposit split (%.2f/deposit), got amount=%v type=%q", total/2, primaryAmount, primaryType)
|
|
}
|
|
// No phantom split rows: the ledger is byte-identical to the sync path's.
|
|
if n := countRound9BookingPayments(t, bookingID); n != paymentsBefore {
|
|
t.Errorf("expected the payment ledger unchanged (%d rows), got %d", paymentsBefore, n)
|
|
}
|
|
// No duplicate balance split: the deterministic key exists exactly once.
|
|
if n := countRound9ByIdempotencyKey(t, idemKey+"-split-1"); n != 1 {
|
|
t.Errorf("expected the balance split key to exist exactly once, got %d", n)
|
|
}
|
|
// Booking side-effects are not double-run: the booking stays 'completed'
|
|
// (a second completion transition cannot fire).
|
|
var bookingStatus string
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil {
|
|
t.Fatalf("failed to read booking status: %v", err)
|
|
}
|
|
if bookingStatus != "completed" {
|
|
t.Errorf("expected booking to stay 'completed', got %q", bookingStatus)
|
|
}
|
|
// The replay is acked and its dedup row committed.
|
|
if n := countWebhookEvents(t, event.EventID); n != 1 {
|
|
t.Errorf("expected 1 dedup row, got %d", n)
|
|
}
|
|
}
|
|
|
|
// TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps locks the flip-guard
|
|
// backstop — the IN-FLIGHT race (fix half 2). The webhook's pending-row SELECT
|
|
// reads the row while it is STILL 'pending' (the sync path's recheck tx holds
|
|
// the booking FOR UPDATE lock but has not committed its flip), then the sync
|
|
// path commits (flip + booking completion) BEFORE the webhook's guarded flip
|
|
// runs. The flip's `AND status='pending'` guard sees the committed 'completed'
|
|
// row and becomes a no-op: the webhook never re-runs the split/completion
|
|
// side-effects, so no duplicate rows and no double booking completion.
|
|
func TestWebhook_Round9_ConcurrentSyncFlip_Wins_FliGuardNoOps(t *testing.T) {
|
|
const (
|
|
squarePaymentID = "sqp_round9_sync_race"
|
|
idemKey = "round9-race-key-1"
|
|
total = 50.00
|
|
)
|
|
payID := createWebhookTestPayment(t, squarePaymentID, "pending")
|
|
bookingID := attachWebhookTestBooking(t, payID, total)
|
|
if _, err := db.Conn.Exec(context.Background(),
|
|
"UPDATE payments SET amount = $1, idempotency_key = $2 WHERE id = $3", total, idemKey, payID); err != nil {
|
|
t.Fatalf("failed to set payment amount/key: %v", err)
|
|
}
|
|
|
|
// The sync path (postChargeRecheck) takes the booking FOR UPDATE lock and
|
|
// holds it while its recheck tx is in flight.
|
|
ctx := context.Background()
|
|
syncTx, err := db.Conn.Begin(ctx)
|
|
if err != nil {
|
|
t.Fatalf("failed to begin sync-path tx: %v", err)
|
|
}
|
|
if _, err := syncTx.Exec(ctx, `SELECT 1 FROM bookings WHERE id = $1 FOR UPDATE`, bookingID); err != nil {
|
|
syncTx.Rollback(ctx)
|
|
t.Fatalf("failed to lock booking row: %v", err)
|
|
}
|
|
|
|
// Deliver the webhook in a goroutine. Signature/body are prepared on the
|
|
// test goroutine (webhookTestEnv calls t.Setenv, which is test-goroutine
|
|
// only); makeWebhookRequest itself touches no *testing.T.
|
|
event := round9CompletedEvent(squarePaymentID, idemKey)
|
|
body, err := json.Marshal(event)
|
|
if err != nil {
|
|
t.Fatalf("failed to marshal webhook event: %v", err)
|
|
}
|
|
sig := webhookTestEnv(t, body)
|
|
done := make(chan *httptest.ResponseRecorder, 1)
|
|
go func() {
|
|
done <- makeWebhookRequest(body, sig, context.Background())
|
|
}()
|
|
|
|
// Wait (bounded) until the webhook tx is actually blocked on the booking
|
|
// FOR UPDATE lock — at that point its pending-row SELECT has ALREADY run and
|
|
// seen the row as 'pending', so the flip-guard interleaving is exercised
|
|
// deterministically. If it never blocks (a slow scheduler), the test still
|
|
// holds: the commit below turns it into the no-op replay interleaving.
|
|
waitDeadline := time.Now().Add(10 * time.Second)
|
|
blocked := false
|
|
for time.Now().Before(waitDeadline) {
|
|
var n int
|
|
if err := db.Conn.QueryRow(ctx, `
|
|
SELECT COUNT(*) FROM pg_stat_activity
|
|
WHERE query ILIKE '%SELECT status FROM bookings WHERE id%'
|
|
AND wait_event_type = 'Lock'
|
|
AND pid <> pg_backend_pid()
|
|
`).Scan(&n); err == nil && n > 0 {
|
|
blocked = true
|
|
break
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
if !blocked {
|
|
t.Logf("webhook never blocked on the booking lock — fell through to the no-op replay interleaving (assertions still hold)")
|
|
}
|
|
|
|
// The sync path commits: guarded flip + booking completion.
|
|
if _, err := syncTx.Exec(ctx,
|
|
`UPDATE payments SET status = 'completed', amount = $1, payment_type = 'deposit', updated_at = NOW() WHERE id = $2`,
|
|
total/2, payID); err != nil {
|
|
syncTx.Rollback(ctx)
|
|
t.Fatalf("failed to flip payment in sync tx: %v", err)
|
|
}
|
|
if _, err := syncTx.Exec(ctx,
|
|
`UPDATE bookings SET status = 'completed', updated_at = NOW() WHERE id = $1`, bookingID); err != nil {
|
|
syncTx.Rollback(ctx)
|
|
t.Fatalf("failed to complete booking in sync tx: %v", err)
|
|
}
|
|
if err := syncTx.Commit(ctx); err != nil {
|
|
t.Fatalf("failed to commit sync-path tx: %v", err)
|
|
}
|
|
|
|
w := <-done
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 (guarded flip no-op), got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
// The payment was completed exactly once — by the sync path.
|
|
if got := getPaymentStatus(t, payID); got != "completed" {
|
|
t.Errorf("expected payment 'completed', got %q", got)
|
|
}
|
|
// No split rows beyond the primary: the webhook's side-effects never ran.
|
|
if n := countRound9BookingPayments(t, bookingID); n != 1 {
|
|
t.Errorf("expected exactly 1 payment row (the sync-flipped primary), got %d", n)
|
|
}
|
|
// The booking was completed by the sync path exactly once.
|
|
var bookingStatus string
|
|
if err := db.Conn.QueryRow(context.Background(),
|
|
"SELECT status FROM bookings WHERE id = $1", bookingID).Scan(&bookingStatus); err != nil {
|
|
t.Fatalf("failed to read booking status: %v", err)
|
|
}
|
|
if bookingStatus != "completed" {
|
|
t.Errorf("expected booking 'completed' (the sync path completed it), got %q", bookingStatus)
|
|
}
|
|
if n := countWebhookEvents(t, event.EventID); n != 1 {
|
|
t.Errorf("expected 1 dedup row, got %d", n)
|
|
}
|
|
}
|