//go:build test && dev package payments // ============================================================================= // ROUND 10 ADVERSARIAL — the till-sale FINAL-key lock vs the sweep's clawback // ============================================================================= // // The sweep (sweep.go acquireTillSaleSweepLock) fails a stale pending // till_sale + claws back its funded gift card under the advisory lock // "crussell:till:". CreateTillSale's base lock is keyed // on the REQUEST key, but a keyless sale's slot scan can advance the FINAL // stored key past the base (a COMPLETED/FAILED sale occupying the base slot // forces "base-1", "base-2", ...). Without the FIX 2 re-acquisition the retry // would hold "crussell:till:" while the sweep holds // "crussell:till:" — two locks that do not serialize — so the sweep can // fail the sale + claw back the funded card while the retry's Square charge is // mid-flight: customer charged AND funding clawed back. // // This test pins the retry side of the invariant: the retry must hold // "crussell:till:" from before the Square charge until it // completes. It seeds a COMPLETED sale on the deterministic base key so the // keyless sale's slot scan advances to the suffixed key, then proves a second // connection attempting the SAME advisory lock is blocked (try-lock returns // false) the whole time the charge is in flight and only acquires after the // retry finishes. import ( "context" "net/http" "net/http/httptest" "testing" "time" "crussell/db" "crussell/internal/square" "crussell/testutils" "crussell/testutils/fixtures" "crussell/testutils/jwt" ) // TestCreateTillSale_KeylessSuffixedSlot_HoldsFinalKeyLockAcrossCharge locks // the FIX 2 retry-side invariant for a keyless sale whose slot scan resolves a // SUFFIXED final idempotency key: // // 1. seed a COMPLETED till_sale occupying the deterministic base key (so the // keyless request's slot scan advances to "-1"); // 2. run the keyless sale with a SLOW Square client (the charge stays // in-flight ~500ms after the pending row commits); // 3. once the pending row with the suffixed key is visible (the tx committed // immediately before the charge), a second connection's try-lock on // "crussell:till:" must return FALSE (the retry holds it) and // stay FALSE while the charge is mid-flight; // 4. after the handler returns, the same try-lock must return TRUE (released) // and the sale must be completed on the suffixed key. // // The lock being held exactly across the charge round-trip is what serializes // against the sweep's "crussell:till:" fail/claw-back. func TestCreateTillSale_KeylessSuffixedSlot_HoldsFinalKeyLockAcrossCharge(t *testing.T) { ctx, tx := testutils.SetupTestTx(t) adminID, err := fixtures.CreateTestAdminUser(tx) if err != nil { t.Fatalf("failed to create admin user: %v", err) } adminToken := jwt.GenerateTestToken(adminID, "admin") pool := context.Background() // The keyless request the test replays. CardToken is deliberately excluded // from deriveTillIdempotencyKey (it changes between retries), so the base // key below is exactly what the handler derives for this request. keylessReq := TillSaleRequest{ ItemType: "gift_card", Action: "create", Amount: 50.00, PaymentMethod: "online_square", CardToken: "cnon:final-lock-adversarial", } baseKey := deriveTillIdempotencyKey(keylessReq, adminID) suffixedKey := nextIdempotencyCandidate(baseKey, 1) // Seed a COMPLETED sale on the base key — it occupies the base slot so the // keyless sale's slot scan must advance to the suffixed key. if _, err := tx.Exec(ctx, ` INSERT INTO till_sales (item_type, description, quantity, unit_price, total_amount, payment_method, status, idempotency_key, created_by, created_at, updated_at) VALUES ('gift_card', 'Gift Card create', 1, 50.00, 50.00, 'online_square', 'completed', $1, $2, NOW(), NOW()) `, baseKey, adminID); err != nil { t.Fatalf("failed to seed the completed base-slot sale: %v", err) } // Cleanup the committed rows (the completed seed + the retry's sale and its // handler-created gift card). t.Cleanup(func() { _, _ = db.Conn.Exec(pool, `DELETE FROM till_sales WHERE idempotency_key IN ($1, $2)`, baseKey, suffixedKey) _, _ = db.Conn.Exec(pool, `DELETE FROM gift_card_transactions WHERE gift_card_id IN (SELECT id FROM gift_cards WHERE created_by = $1)`, adminID) _, _ = db.Conn.Exec(pool, `DELETE FROM gift_cards WHERE created_by = $1`, adminID) _, _ = db.Conn.Exec(pool, `DELETE FROM users WHERE id = $1`, adminID) }) // Commit the setup so the handler and the lock-checking connection operate // at pool level on independent sessions — advisory locks only serialize // across separate sessions, and a per-test tx would mask the contention. innerTx := db.TxFromContext(ctx) if innerTx == nil { t.Fatal("no transaction in context") } if err := innerTx.Commit(ctx); err != nil { t.Fatalf("failed to commit setup tx: %v", err) } // Slow the Square charge so the handler provably holds the final-key lock // across an in-flight round-trip long enough to observe it from another // connection. origClient := SquareClient slow := &slowCreatePaymentClient{SquareClient: square.NewDevClient(), delay: 500 * time.Millisecond} SquareClient = slow defer func() { SquareClient = origClient }() // Run the keyless sale in a goroutine — the handler blocks ~500ms inside // the Square charge and must not block this test's lock observations. recCh := make(chan *httptest.ResponseRecorder, 1) go func() { recCh <- makePaymentRequest(CreateTillSale, "POST", "/api/admin/till/sale", keylessReq, adminToken, pool) }() // Wait for the pending till_sales row on the SUFFIXED key to be committed — // the tx commit precedes the Square charge, so once it is visible the // handler holds the final-key lock and is about to call Square. deadline := time.Now().Add(10 * time.Second) for { var status string err := db.Conn.QueryRow(pool, `SELECT status FROM till_sales WHERE idempotency_key = $1`, suffixedKey).Scan(&status) if err == nil { break } if time.Now().After(deadline) { t.Fatalf("timed out waiting for the pending till_sale on suffixed key %s (err: %v)", suffixedKey, err) } time.Sleep(10 * time.Millisecond) } lockConn, err := db.Conn.Acquire(pool) if err != nil { t.Fatalf("failed to acquire lock-check connection: %v", err) } defer lockConn.Release() tryLock := func() bool { t.Helper() var acquired bool if err := lockConn.QueryRow(pool, `SELECT pg_try_advisory_lock(hashtext($1))`, "crussell:till:"+suffixedKey).Scan(&acquired); err != nil { t.Fatalf("failed to try the final-key advisory lock: %v", err) } return acquired } // The charge is mid-flight: the retry must hold the final-key lock, so a // second session cannot acquire it. If the lock is free the retry is NOT // serialized against the sweep's stored-key clawback — the exact FIX 2 bug. if tryLock() { t.Fatalf("retry does NOT hold the final-key lock %q while the Square charge is in flight — the sweep could fail the sale and claw back the funded card mid-charge", "crussell:till:"+suffixedKey) } // Still mid-flight (the slow client keeps the charge in the air for ~500ms // after the row appeared): the lock must stay held for the whole window. time.Sleep(200 * time.Millisecond) if tryLock() { t.Fatalf("final-key lock %q was released mid-charge — the sweep could claw back the funded card before the charge completed", "crussell:till:"+suffixedKey) } // The handler must complete the sale on the suffixed key. var w *httptest.ResponseRecorder select { case w = <-recCh: case <-time.After(30 * time.Second): t.Fatal("timed out waiting for the till-sale handler") } if w.Code != http.StatusCreated { t.Fatalf("expected 201, got %d. body: %s", w.Code, w.Body.String()) } // After the handler released it, the final-key lock must be free again — // proving it was held for exactly the charge round-trip. if !tryLock() { t.Fatalf("final-key lock %q not released after the retry completed", "crussell:till:"+suffixedKey) } defer func() { _, _ = lockConn.Exec(context.Background(), `SELECT pg_advisory_unlock(hashtext($1))`, "crussell:till:"+suffixedKey) }() // The sale is stored under the SUFFIXED key and completed — the key the // sweep will lock on, and the one this retry just held. var storedKey, status string if err := db.Conn.QueryRow(pool, `SELECT idempotency_key, status FROM till_sales WHERE idempotency_key = $1`, suffixedKey).Scan(&storedKey, &status); err != nil { t.Fatalf("failed to query the suffixed-key sale: %v", err) } if storedKey != suffixedKey { t.Fatalf("stored idempotency_key %q != suffixed key %q", storedKey, suffixedKey) } if status != "completed" { t.Fatalf("expected the suffixed-key sale to be completed, got %q", status) } // The base slot stays occupied by the seeded completed sale (the keyless // retry did not collapse onto it). var baseCount int if err := db.Conn.QueryRow(pool, `SELECT COUNT(*) FROM till_sales WHERE idempotency_key = $1 AND status = 'completed'`, baseKey).Scan(&baseCount); err != nil { t.Fatalf("failed to count base-slot sales: %v", err) } if baseCount != 1 { t.Fatalf("expected the base-slot sale to remain the only completed sale on the base key, got %d", baseCount) } }