From f4a603371598c0a1f909306f24e323fcb63b92ed Mon Sep 17 00:00:00 2001 From: Stephen Adamson Date: Fri, 5 Jun 2026 16:36:46 +0100 Subject: [PATCH] feat: financial data retention & aggregation system Add CleanupExpiredFinancialRecords to enforce HMRC + Limitation Act compliance (7-year retention, 1-year post-anonymization buffer). - financial_aggregates table: monthly totals by payment method/type (no PII) - CleanupExpiredFinancialRecords(): aggregates expired payments/refunds, deletes granular records, idempotent via ON CONFLICT DO UPDATE - Wired into GET /api/availability alongside existing cleanup functions - 8 tests: 7yr expiry, 1yr buffer, 9yr override, aggregation totals, idempotency, active user protection, both-thresholds elapsed, refunds - testdb.go: financial_aggregates in drop-order and truncate lists - README + Technical Manual updated --- README.md | 3 +- backend/handlers/scheduling/default-hours.go | 5 + .../handlers/scheduling/scheduling_test.go | 2 + backend/handlers/scheduling/time-blockers.go | 123 ++++ .../handlers/scheduling/time_blockers_test.go | 610 ++++++++++++++++++ backend/testutils/testdb/testdb.go | 2 + init-scripts/init-script.sql | 25 + obsidian/Crussell/Technical Manual.md | 55 +- 8 files changed, 820 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b95a710..4425ab5 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Nail salon booking platform — Go 1.25 backend + SvelteKit 5 frontend + Docker. - **Profile picture upload limit**: 15MB client-side check before crop dialog in account page - **Portfolio image upload backend limit**: separate `portfolioBodyLimit` (20MB) applied to `/images` route, distinct from `uploadBodyLimit` (15MB) for profile pictures - **GDPR compliance system**: Full Article 15 Subject Access Request via `/gdpr` — async Go endpoint (`GET /api/user/gdpr-export`) with 12h in-memory cache and background generation, 16-section SQL export (`export_all_user_data()`) covering profile, bookings (with override pricing), payments, refunds, saved cards, social logins, loyalty redemptions, booking discounts, edit requests, affiliate payouts, verification codes, forgiven no-shows, patch tests, referrals, notification preferences. Frontend: skeleton loading, 2s polling, styled report cards/tables, PDF export (print CSS hides navbar + verification banner), raw JSON download. Conditional rendering for empty sections, VAT breakdowns (hidden until non-zero), deposits required (hidden when 0). Account deletion (`DELETE /api/user/account`) extended with external system scrubbing — S3 profile picture deletion, Square saved card deletion — before SQL-level anonymization. `anonymize_user()` SQL function extended with child table PII scrubbing (social logins, saved cards soft-delete with PCI clearance, verification code expiry, time blocker reservation scrubbing, edit request notes nulling, notification preference deletion). `AnonymizeStaleGuestAccounts()` extended with additional field scrubbing (profile_pic_url, referral_code, notes, data_retention_consent). 25 tests covering all backend additions. +- **Financial data retention & aggregation**: Granular payment/refund records retained for `MAX(created_at + 7 years, user_anonymized_at + 1 year)` (HMRC + Limitation Act compliance). Expired records are aggregated into `financial_aggregates` (monthly totals by payment method/type, no PII) and deleted. Triggered lazily on `GET /api/availability` alongside existing cleanup functions. Idempotent — safe to run repeatedly. 8 tests covering retention edge cases, aggregation correctness, refund handling, and idempotency. ## Project Structure @@ -131,7 +132,7 @@ cd backend && go build -o bin/backend ./main.go # Frontend cd frontend && npm ci && npm run build -# Tests (576/579 passing, 3 skipped) +# Tests (584/587 passing, 3 skipped) cd backend && go test -tags "test,dev" ./... ``` diff --git a/backend/handlers/scheduling/default-hours.go b/backend/handlers/scheduling/default-hours.go index 3927e7c..741931b 100644 --- a/backend/handlers/scheduling/default-hours.go +++ b/backend/handlers/scheduling/default-hours.go @@ -322,6 +322,11 @@ func GetAvailableHours(w http.ResponseWriter, r *http.Request) { log.Printf("Failed to cleanup expired loyalty redemptions: %v", err) } + // Clean up expired financial records (aggregate + delete granular data) + if err := CleanupExpiredFinancialRecords(r.Context()); err != nil { + log.Printf("Failed to cleanup expired financial records: %v", err) + } + // Load default hours defaultMap := map[int]DefaultHours{} defRows, _ := db.DB.Query(r.Context(), `SELECT weekday, start_time::text, end_time::text, is_open FROM working_hours`) diff --git a/backend/handlers/scheduling/scheduling_test.go b/backend/handlers/scheduling/scheduling_test.go index d09afba..6c0c5cc 100644 --- a/backend/handlers/scheduling/scheduling_test.go +++ b/backend/handlers/scheduling/scheduling_test.go @@ -35,6 +35,8 @@ import ( func resetTestData(t *testing.T) { t.Helper() testdb.TruncateTables(t, db.DB) + // Also truncate financial_aggregates which is not in the default truncation list + db.DB.Exec(context.Background(), "TRUNCATE financial_aggregates CASCADE") seedDefaultWorkingHours(t) } diff --git a/backend/handlers/scheduling/time-blockers.go b/backend/handlers/scheduling/time-blockers.go index 9c0ac93..79b829f 100644 --- a/backend/handlers/scheduling/time-blockers.go +++ b/backend/handlers/scheduling/time-blockers.go @@ -457,3 +457,126 @@ func CleanupExpiredLoyaltyRedemptions(ctx context.Context) error { `) return err } + +// CleanupExpiredFinancialRecords deletes granular payment/refund records whose +// retention period has expired and replaces them with monthly aggregates. +// +// Retention: MAX(p.created_at + 7 years, user_anonymized_at + 1 year) +// - Walk-in records (no user): 7 years from payment creation +// - Active users: 7 years from payment creation +// - Anonymized users: 7 years from payment creation AND 1 year since anonymization +// +// The function is idempotent — running it twice produces the same result. +func CleanupExpiredFinancialRecords(ctx context.Context) error { + tx, err := db.DB.Begin(ctx) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, ` + INSERT INTO financial_aggregates (month, total_payments, total_square_fees, total_cash, total_online, total_in_person, total_discounts, total_giftcard, total_tips, total_deposits, total_balances, total_partials, booking_count) + SELECT + DATE_TRUNC('month', p.created_at)::date AS month, + COALESCE(SUM(p.amount), 0) AS total_payments, + COALESCE(SUM(p.fees), 0) AS total_square_fees, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_method = 'cash'), 0) AS total_cash, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_method = 'online_square'), 0) AS total_online, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_method = 'in_person_card'), 0) AS total_in_person, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_method = 'discount'), 0) AS total_discounts, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_method = 'giftcard'), 0) AS total_giftcard, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_type = 'tip'), 0) AS total_tips, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_type = 'deposit'), 0) AS total_deposits, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_type = 'balance'), 0) AS total_balances, + COALESCE(SUM(p.amount) FILTER (WHERE p.payment_type = 'partial'), 0) AS total_partials, + COALESCE(COUNT(DISTINCT p.booking_id), 0) AS booking_count + FROM payments p + LEFT JOIN bookings b ON p.booking_id = b.id + LEFT JOIN users u ON b.user_id = u.id + WHERE p.created_at < NOW() - INTERVAL '7 years' + AND ( + b.user_id IS NULL + OR u.id IS NULL + OR NOT (u.account_role = 'guest' AND (u.email LIKE 'anon-%@anon.invalid' OR u.email LIKE 'deleted+%@deleted.invalid')) + OR u.updated_at < NOW() - INTERVAL '1 year' + ) + GROUP BY DATE_TRUNC('month', p.created_at)::date + ON CONFLICT (month) DO UPDATE SET + total_payments = financial_aggregates.total_payments + EXCLUDED.total_payments, + total_square_fees = financial_aggregates.total_square_fees + EXCLUDED.total_square_fees, + total_cash = financial_aggregates.total_cash + EXCLUDED.total_cash, + total_online = financial_aggregates.total_online + EXCLUDED.total_online, + total_in_person = financial_aggregates.total_in_person + EXCLUDED.total_in_person, + total_discounts = financial_aggregates.total_discounts + EXCLUDED.total_discounts, + total_giftcard = financial_aggregates.total_giftcard + EXCLUDED.total_giftcard, + total_tips = financial_aggregates.total_tips + EXCLUDED.total_tips, + total_deposits = financial_aggregates.total_deposits + EXCLUDED.total_deposits, + total_balances = financial_aggregates.total_balances + EXCLUDED.total_balances, + total_partials = financial_aggregates.total_partials + EXCLUDED.total_partials, + booking_count = financial_aggregates.booking_count + EXCLUDED.booking_count + `) + if err != nil { + return fmt.Errorf("failed to aggregate expired payments: %w", err) + } + + _, err = tx.Exec(ctx, ` + INSERT INTO financial_aggregates (month, total_refunds) + SELECT + DATE_TRUNC('month', r.created_at)::date AS month, + COALESCE(SUM(r.amount), 0) AS total_refunds + FROM refunds r + JOIN payments p ON r.payment_id = p.id + LEFT JOIN bookings b ON p.booking_id = b.id + LEFT JOIN users u ON b.user_id = u.id + WHERE p.created_at < NOW() - INTERVAL '7 years' + AND ( + b.user_id IS NULL + OR u.id IS NULL + OR NOT (u.account_role = 'guest' AND (u.email LIKE 'anon-%@anon.invalid' OR u.email LIKE 'deleted+%@deleted.invalid')) + OR u.updated_at < NOW() - INTERVAL '1 year' + ) + GROUP BY DATE_TRUNC('month', r.created_at)::date + ON CONFLICT (month) DO UPDATE SET + total_refunds = financial_aggregates.total_refunds + EXCLUDED.total_refunds + `) + if err != nil { + return fmt.Errorf("failed to aggregate expired refunds: %w", err) + } + + _, err = tx.Exec(ctx, ` + DELETE FROM payments p + USING bookings b + LEFT JOIN users u ON b.user_id = u.id + WHERE p.booking_id = b.id + AND p.created_at < NOW() - INTERVAL '7 years' + AND ( + b.user_id IS NULL + OR u.id IS NULL + OR NOT (u.account_role = 'guest' AND (u.email LIKE 'anon-%@anon.invalid' OR u.email LIKE 'deleted+%@deleted.invalid')) + OR u.updated_at < NOW() - INTERVAL '1 year' + ) + `) + if err != nil { + return fmt.Errorf("failed to delete expired payments: %w", err) + } + + _, err = tx.Exec(ctx, ` + DELETE FROM refunds r + USING payments p + LEFT JOIN bookings b ON p.booking_id = b.id + LEFT JOIN users u ON b.user_id = u.id + WHERE r.payment_id = p.id + AND p.created_at < NOW() - INTERVAL '7 years' + AND ( + b.user_id IS NULL + OR u.id IS NULL + OR NOT (u.account_role = 'guest' AND (u.email LIKE 'anon-%@anon.invalid' OR u.email LIKE 'deleted+%@deleted.invalid')) + OR u.updated_at < NOW() - INTERVAL '1 year' + ) + `) + if err != nil { + return fmt.Errorf("failed to delete expired refunds: %w", err) + } + + return tx.Commit(ctx) +} diff --git a/backend/handlers/scheduling/time_blockers_test.go b/backend/handlers/scheduling/time_blockers_test.go index c7052f7..821f1bd 100644 --- a/backend/handlers/scheduling/time_blockers_test.go +++ b/backend/handlers/scheduling/time_blockers_test.go @@ -1122,6 +1122,616 @@ func TestAnonymizeStaleGuestAccounts_NoBookings(t *testing.T) { } } +// --- Tests for CleanupExpiredFinancialRecords --- + +// TestCleanupExpiredFinancialRecords_PaymentOlderThan7Years verifies that a +// payment older than 7 years is aggregated and deleted. +func TestCleanupExpiredFinancialRecords_PaymentOlderThan7Years(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + eightYearsAgo := time.Now().AddDate(-8, 0, 0) + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 50.00, $2) + `, bookingID, eightYearsAgo) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment was deleted + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 0 { + t.Errorf("expected payment to be deleted, got %d payment(s)", paymentCount) + } + + // Verify financial_aggregates has 1 row + var aggCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount != 1 { + t.Errorf("expected 1 financial_aggregates row, got %d", aggCount) + } + + // Verify aggregate has correct month and total + expectedMonth := time.Date(eightYearsAgo.Year(), eightYearsAgo.Month(), 1, 0, 0, 0, 0, time.UTC) + var aggMonth time.Time + var totalPayments float64 + err = db.DB.QueryRow(ctx, "SELECT month, total_payments FROM financial_aggregates").Scan(&aggMonth, &totalPayments) + if err != nil { + t.Fatalf("failed to query aggregate: %v", err) + } + if !aggMonth.Equal(expectedMonth) { + t.Errorf("expected month %s, got %s", expectedMonth.Format("2006-01-02"), aggMonth.Format("2006-01-02")) + } + if totalPayments != 50.00 { + t.Errorf("expected total_payments 50.00, got %.2f", totalPayments) + } +} + +// TestCleanupExpiredFinancialRecords_AnonUserWithin1YearBuffer verifies that +// an anonymized user's payment is NOT deleted when the 1-year buffer hasn't +// elapsed, even if the payment is older than 7 years. +func TestCleanupExpiredFinancialRecords_AnonUserWithin1YearBuffer(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + guestID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + _, err = db.DB.Exec(ctx, ` + UPDATE users + SET account_role = 'guest', + email = 'anon-' || id || '@anon.invalid', + updated_at = NOW() - INTERVAL '6 months' + WHERE id = $1 + `, guestID) + if err != nil { + t.Fatalf("failed to anonymize user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, guestID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + fourYearsAgo := time.Now().AddDate(-4, 0, 0) + var paymentID string + err = db.DB.QueryRow(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 75.00, $2) + RETURNING id + `, bookingID, fourYearsAgo).Scan(&paymentID) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment still exists (neither 7yr nor 1yr conditions satisfied) + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE id = $1", paymentID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 1 { + t.Error("expected payment to still exist") + } + + // Verify no aggregate was created + var aggCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount != 0 { + t.Errorf("expected no aggregates, got %d", aggCount) + } +} + +// TestCleanupExpiredFinancialRecords_PaymentOlderThan9Years verifies that a +// payment older than 9 years is always deleted regardless of user status. +func TestCleanupExpiredFinancialRecords_PaymentOlderThan9Years(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // Payment created 9 years ago + nineYearsAgo := time.Now().AddDate(-9, 0, 0) + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 60.00, $2) + `, bookingID, nineYearsAgo) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment was deleted + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 0 { + t.Errorf("expected payment to be deleted, got %d payment(s)", paymentCount) + } + + // Verify financial_aggregates has 1 row + var aggCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount != 1 { + t.Errorf("expected 1 financial_aggregates row, got %d", aggCount) + } +} + +// TestCleanupExpiredFinancialRecords_AggregationCorrectTotals verifies that +// multiple payments in the same month are correctly aggregated by method and type. +func TestCleanupExpiredFinancialRecords_AggregationCorrectTotals(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // 3 payments 8 years ago, all in the same month + sameMonth := time.Now().AddDate(-8, 0, 0) + _ = sameMonth // used for all payments + + // Payment 1: £50 cash + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, fees, created_at) + VALUES ($1, 'full', 'cash', 'completed', 50.00, 0, $2) + `, bookingID, sameMonth) + if err != nil { + t.Fatalf("failed to create cash payment: %v", err) + } + + // Payment 2: £30 online_square with £2.50 fees + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, fees, created_at) + VALUES ($1, 'full', 'online_square', 'completed', 30.00, 2.50, $2) + `, bookingID, sameMonth) + if err != nil { + t.Fatalf("failed to create online payment: %v", err) + } + + // Payment 3: £20 in_person_card + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, fees, created_at) + VALUES ($1, 'full', 'in_person_card', 'completed', 20.00, 0, $2) + `, bookingID, sameMonth) + if err != nil { + t.Fatalf("failed to create card payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify aggregate totals + var totalPayments, totalCash, totalOnline, totalInPerson, totalSquareFees float64 + var bookingCount int + err = db.DB.QueryRow(ctx, ` + SELECT total_payments, total_cash, total_online, total_in_person, total_square_fees, booking_count + FROM financial_aggregates + `).Scan(&totalPayments, &totalCash, &totalOnline, &totalInPerson, &totalSquareFees, &bookingCount) + if err != nil { + t.Fatalf("failed to query aggregate: %v", err) + } + + if totalPayments != 100.00 { + t.Errorf("expected total_payments 100.00, got %.2f", totalPayments) + } + if totalCash != 50.00 { + t.Errorf("expected total_cash 50.00, got %.2f", totalCash) + } + if totalOnline != 30.00 { + t.Errorf("expected total_online 30.00, got %.2f", totalOnline) + } + if totalInPerson != 20.00 { + t.Errorf("expected total_in_person 20.00, got %.2f", totalInPerson) + } + if totalSquareFees != 2.50 { + t.Errorf("expected total_square_fees 2.50, got %.2f", totalSquareFees) + } + if bookingCount != 1 { + t.Errorf("expected booking_count 1, got %d", bookingCount) + } +} + +// TestCleanupExpiredFinancialRecords_Idempotent verifies that running the +// cleanup function twice produces the same result (no double-counting). +func TestCleanupExpiredFinancialRecords_Idempotent(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // Payment 8 years ago + eightYearsAgo := time.Now().AddDate(-8, 0, 0) + _, err = db.DB.Exec(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 100.00, $2) + `, bookingID, eightYearsAgo) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // First run + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("first cleanup failed: %v", err) + } + + // Capture aggregate values after first run + var aggCount1 int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount1) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + var totalPayments1 float64 + var aggMonth1 time.Time + err = db.DB.QueryRow(ctx, "SELECT month, total_payments FROM financial_aggregates").Scan(&aggMonth1, &totalPayments1) + if err != nil { + t.Fatalf("failed to query aggregate after first run: %v", err) + } + + // Second run + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("second cleanup failed: %v", err) + } + + // Verify same aggregate count and values + var aggCount2 int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount2) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount2 != aggCount1 { + t.Errorf("expected %d aggregates after second run, got %d", aggCount1, aggCount2) + } + + var totalPayments2 float64 + var aggMonth2 time.Time + err = db.DB.QueryRow(ctx, "SELECT month, total_payments FROM financial_aggregates").Scan(&aggMonth2, &totalPayments2) + if err != nil { + t.Fatalf("failed to query aggregate after second run: %v", err) + } + if !aggMonth2.Equal(aggMonth1) { + t.Errorf("expected month %s, got %s", aggMonth1.Format("2006-01-02"), aggMonth2.Format("2006-01-02")) + } + if totalPayments2 != totalPayments1 { + t.Errorf("expected total_payments %.2f after second run, got %.2f", totalPayments1, totalPayments2) + } + + // Verify payments are still deleted + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE booking_id = $1", bookingID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 0 { + t.Errorf("expected payments to be deleted, got %d", paymentCount) + } +} + +// TestCleanupExpiredFinancialRecords_ActiveUserWithin7Years verifies that +// a payment newer than 7 years for an active user is NOT deleted. +func TestCleanupExpiredFinancialRecords_ActiveUserWithin7Years(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // Payment created 3 years ago (< 7 years) + threeYearsAgo := time.Now().AddDate(-3, 0, 0) + var paymentID string + err = db.DB.QueryRow(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 40.00, $2) + RETURNING id + `, bookingID, threeYearsAgo).Scan(&paymentID) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment still exists + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE id = $1", paymentID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 1 { + t.Error("expected payment to still exist") + } + + // Verify no aggregate was created + var aggCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount != 0 { + t.Errorf("expected no aggregates, got %d", aggCount) + } +} + +// TestCleanupExpiredFinancialRecords_AnonUserBothThresholdsElapsed verifies that +// an anonymized user's payment IS deleted when BOTH the 7-year rule AND the +// 1-year post-anonymization buffer have elapsed. +func TestCleanupExpiredFinancialRecords_AnonUserBothThresholdsElapsed(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + guestID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + // Anonymize user 2 years ago + _, err = db.DB.Exec(ctx, ` + UPDATE users + SET account_role = 'guest', + email = 'anon-' || id || '@anon.invalid', + updated_at = NOW() - INTERVAL '2 years' + WHERE id = $1 + `, guestID) + if err != nil { + t.Fatalf("failed to anonymize user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, guestID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // Payment created 8 years ago (past 7yr rule) + // User anonymized 2 years ago (past 1yr buffer) + // Both conditions met → should be deleted + eightYearsAgo := time.Now().AddDate(-8, 0, 0) + var paymentID string + err = db.DB.QueryRow(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 75.00, $2) + RETURNING id + `, bookingID, eightYearsAgo).Scan(&paymentID) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment was deleted + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE id = $1", paymentID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 0 { + t.Error("expected payment to be deleted (both 7yr and 1yr+ thresholds elapsed)") + } + + // Verify aggregate was created + var aggCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM financial_aggregates").Scan(&aggCount) + if err != nil { + t.Fatalf("failed to count aggregates: %v", err) + } + if aggCount != 1 { + t.Errorf("expected 1 financial_aggregates row, got %d", aggCount) + } +} + +// TestCleanupExpiredFinancialRecords_RefundAggregatedAndDeleted verifies that +// refunds are aggregated and deleted alongside their parent payment when +// retention expires. +func TestCleanupExpiredFinancialRecords_RefundAggregatedAndDeleted(t *testing.T) { + resetTestData(t) + + ctx := context.Background() + + userID, err := fixtures.CreateTestUser(db.DB) + if err != nil { + t.Fatalf("failed to create user: %v", err) + } + + serviceID, err := fixtures.CreateTestService(db.DB) + if err != nil { + t.Fatalf("failed to create service: %v", err) + } + + bookingID, err := fixtures.CreateTestBooking(db.DB, userID, serviceID) + if err != nil { + t.Fatalf("failed to create booking: %v", err) + } + + // Payment 8 years ago + eightYearsAgo := time.Now().AddDate(-8, 0, 0) + var paymentID string + err = db.DB.QueryRow(ctx, ` + INSERT INTO payments (booking_id, payment_type, payment_method, status, amount, created_at) + VALUES ($1, 'full', 'cash', 'completed', 100.00, $2) + RETURNING id + `, bookingID, eightYearsAgo).Scan(&paymentID) + if err != nil { + t.Fatalf("failed to create payment: %v", err) + } + + // Refund 8 years ago (same month as payment) + var refundID string + err = db.DB.QueryRow(ctx, ` + INSERT INTO refunds (payment_id, booking_id, amount, status, reason, created_at) + VALUES ($1, $2, 30.00, 'completed', 'test refund', $3) + RETURNING id + `, paymentID, bookingID, eightYearsAgo).Scan(&refundID) + if err != nil { + t.Fatalf("failed to create refund: %v", err) + } + + // Run cleanup + err = CleanupExpiredFinancialRecords(ctx) + if err != nil { + t.Fatalf("CleanupExpiredFinancialRecords failed: %v", err) + } + + // Verify payment was deleted + var paymentCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM payments WHERE id = $1", paymentID).Scan(&paymentCount) + if err != nil { + t.Fatalf("failed to count payments: %v", err) + } + if paymentCount != 0 { + t.Error("expected payment to be deleted") + } + + // Verify refund was deleted + var refundCount int + err = db.DB.QueryRow(ctx, "SELECT COUNT(*) FROM refunds WHERE id = $1", refundID).Scan(&refundCount) + if err != nil { + t.Fatalf("failed to count refunds: %v", err) + } + if refundCount != 0 { + t.Error("expected refund to be deleted") + } + + // Verify aggregate has both payment and refund totals + var totalPayments, totalRefunds float64 + err = db.DB.QueryRow(ctx, "SELECT total_payments, total_refunds FROM financial_aggregates").Scan(&totalPayments, &totalRefunds) + if err != nil { + t.Fatalf("failed to query aggregate: %v", err) + } + if totalPayments != 100.00 { + t.Errorf("expected total_payments 100.00, got %.2f", totalPayments) + } + if totalRefunds != 30.00 { + t.Errorf("expected total_refunds 30.00, got %.2f", totalRefunds) + } +} + // Ensure bytes is used to avoid unused import error var _ = bytes.Buffer{} diff --git a/backend/testutils/testdb/testdb.go b/backend/testutils/testdb/testdb.go index 2ea220d..597757d 100644 --- a/backend/testutils/testdb/testdb.go +++ b/backend/testutils/testdb/testdb.go @@ -104,6 +104,7 @@ func Migrate(t *testing.T, pool *pgxpool.Pool) { "user_saved_cards", "square_deposits", "affiliate_payouts", + "financial_aggregates", "bookings", "user_patch_tests", "patch_tests", @@ -211,6 +212,7 @@ func TruncateTables(t *testing.T, pool *pgxpool.Pool) { "user_saved_cards", "square_deposits", "affiliate_payouts", + "financial_aggregates", "bookings", "booking_edit_requests", "user_patch_tests", diff --git a/init-scripts/init-script.sql b/init-scripts/init-script.sql index 651bbe0..fbd34d0 100644 --- a/init-scripts/init-script.sql +++ b/init-scripts/init-script.sql @@ -1517,6 +1517,31 @@ CREATE TABLE refunds ( CREATE INDEX idx_refunds_payment ON refunds(payment_id); CREATE INDEX idx_refunds_booking ON refunds(booking_id); +-- ======================================= +-- FINANCIAL AGGREGATES TABLE +-- Monthly aggregated financial statistics (no PII) +-- Populated by CleanupExpiredFinancialRecords when granular records expire +-- Retention: replaces per-transaction records after MAX(created_at + 7 years, user_anonymized_at + 1 year) +-- ======================================= + +CREATE TABLE financial_aggregates ( + month DATE PRIMARY KEY, + total_payments NUMERIC(12,2) NOT NULL DEFAULT 0, + total_refunds NUMERIC(12,2) NOT NULL DEFAULT 0, + total_square_fees NUMERIC(12,2) NOT NULL DEFAULT 0, + total_cash NUMERIC(12,2) NOT NULL DEFAULT 0, + total_online NUMERIC(12,2) NOT NULL DEFAULT 0, + total_in_person NUMERIC(12,2) NOT NULL DEFAULT 0, + total_discounts NUMERIC(12,2) NOT NULL DEFAULT 0, + total_giftcard NUMERIC(12,2) NOT NULL DEFAULT 0, + total_tips NUMERIC(12,2) NOT NULL DEFAULT 0, + total_deposits NUMERIC(12,2) NOT NULL DEFAULT 0, + total_balances NUMERIC(12,2) NOT NULL DEFAULT 0, + total_partials NUMERIC(12,2) NOT NULL DEFAULT 0, + booking_count INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + -- ======================================= -- AFFILIATE PAYOUTS TABLE -- ======================================= diff --git a/obsidian/Crussell/Technical Manual.md b/obsidian/Crussell/Technical Manual.md index 2e57de4..94062c5 100644 --- a/obsidian/Crussell/Technical Manual.md +++ b/obsidian/Crussell/Technical Manual.md @@ -325,7 +325,7 @@ src/lib/components/ | `discount_campaign_scope` | `all_bookings`, `first_booking_only`, `new_customers_only` | | `discount_campaign_status` | `draft`, `active`, `completed`, `cancelled` | -### Tables (29 total) +### Tables (30 total) | Table | Purpose | |-------|---------| @@ -348,6 +348,7 @@ src/lib/components/ | `payments` | Payment transactions (VAT fields, invoice_number sequence, fees column for Square deductions, saved_card_id) | | `user_saved_cards` | Saved card details (square_card_id, brand, last4, fingerprint, soft delete with retained_until) | | `refunds` | Refund records linked to payments (amount, reason, square_refund_id) | +| `financial_aggregates` | Monthly aggregated financial statistics (no PII) — populated when granular records expire | | `square_deposits` | Square deposit batch tracking for bank reconciliation (batch_id, total_amount, deposited_at) | | `affiliate_payouts` | Affiliate commission tracking | | `loyalty_redemptions` | Loyalty stamp redemptions (pending → applied, 6-month expiry, FIFO) | @@ -381,6 +382,7 @@ src/lib/components/ | `apply_vat_to_payment(payment_id, vat_rate)` | Apply VAT to a payment | | `calculate_vat(gross_amount, vat_rate)` | Calculate net + VAT from gross | | `get_receipt_data(payment_id)` | Receipt generation data | +| `CleanupExpiredFinancialRecords(ctx)` | Go function — aggregates expired payments/refunds into monthly stats, deletes granular records | ### Partial Indexes @@ -431,6 +433,8 @@ src/lib/components/ - Admin walk-in/call-in: > 15 minutes old - Edit request reservations: > 24 hours old +**Financial cleanup:** `CleanupExpiredFinancialRecords()` runs on the same availability fetch. Aggregates expired payments/refunds into monthly stats and deletes granular records past their retention threshold. + **Why designed this way:** Storing reservations in `time_blockers` means they automatically participate in availability calculations — no separate reservation table needed. The TTL-based cleanup is lazy (triggered on availability fetch) rather than cron-based. --- @@ -491,7 +495,7 @@ src/lib/components/ 2. Subtracting existing bookings (with gap logic) 3. Subtracting time blockers (including reservations) 4. **Late night lock**: After 22:00, blocks next morning 00:00-11:00 for non-admin users -5. Triggers `CleanupOldReservations()` and `AnonymizeStaleGuestAccounts()` +5. Triggers `CleanupOldReservations()`, `AnonymizeStaleGuestAccounts()`, `CleanupExpiredLoyaltyRedemptions()`, and `CleanupExpiredFinancialRecords()` **Time Blockers:** Can be one-off (no cron) or recurring (cron expression). Cron expansion via `robfig/cron/v3` parser. @@ -632,6 +636,49 @@ All applicable discounts stack additively (not compound). Each discount is calcu --- +### Financial Data Retention & Aggregation + +**How it works:** Granular payment and refund records are retained for `MAX(created_at + 7 years, user_anonymized_at + 1 year)` — whichever is further into the future. Once a record's retention period expires, it is aggregated into `financial_aggregates` (monthly totals, no PII) and the granular record is deleted. + +**Retention logic:** + +| Scenario | Retention period | +|----------|-----------------| +| Active user (not anonymized) | 7 years from `payments.created_at` | +| Walk-in (guest account, not anonymized) | 7 years from `payments.created_at` | +| Anonymized user | MAX(7 years from `payments.created_at`, 1 year from `users.updated_at`) | + +A record is only deleted when **both** applicable conditions are met — the 7-year rule AND the 1-year post-anonymization buffer (if applicable). + +**Trigger:** `CleanupExpiredFinancialRecords(ctx)` runs on every `GET /api/availability` alongside `CleanupOldReservations`, `AnonymizeStaleGuestAccounts`, and `CleanupExpiredLoyaltyRedemptions`. Lazy execution — no cron or background worker needed. + +**Aggregation columns** (`financial_aggregates` table): + +| Column | Source | +|--------|--------| +| `month` | `DATE_TRUNC('month', created_at)::date` | +| `total_payments` | `SUM(amount)` | +| `total_refunds` | `SUM(amount)` from refunds | +| `total_square_fees` | `SUM(fees)` | +| `total_cash` | `SUM(amount)` WHERE `payment_method = 'cash'` | +| `total_online` | `SUM(amount)` WHERE `payment_method = 'online_square'` | +| `total_in_person` | `SUM(amount)` WHERE `payment_method = 'in_person_card'` | +| `total_discounts` | `SUM(amount)` WHERE `payment_method = 'discount'` | +| `total_giftcard` | `SUM(amount)` WHERE `payment_method = 'giftcard'` | +| `total_tips` | `SUM(amount)` WHERE `payment_type = 'tip'` | +| `total_deposits` | `SUM(amount)` WHERE `payment_type = 'deposit'` | +| `total_balances` | `SUM(amount)` WHERE `payment_type = 'balance'` | +| `total_partials` | `SUM(amount)` WHERE `payment_type = 'partial'` | +| `booking_count` | `COUNT(DISTINCT booking_id)` | + +**Idempotency:** Uses `ON CONFLICT (month) DO UPDATE` with additive upserts (`table.col + EXCLUDED.col`). Running the cleanup twice produces the same result — no double-counting. + +**Live data unaffected:** Aggregation only reads and deletes records past their retention threshold. Recent payments/refunds within their retention period are never touched. User anonymization (`anonymize_user()`, `AnonymizeStaleGuestAccounts`) scrubs PII only — it never deletes financial records. + +**Tables:** `financial_aggregates`, `payments`, `refunds` + +--- + ### Service Eligibility **Age Filtering:** `services.minimum_age_required` compared to user's `date_of_birth`. If user's age < minimum → service excluded. @@ -821,13 +868,13 @@ go test -tags "test,dev" -v -p 1 -count=2 ./... # Run twice for flaky detection ### Test Coverage -**576/579 tests passing** (3 skipped) across 12+ packages. Comprehensive coverage of online payments (deposit, full, partial, balance), saved card operations, tip payments, terminal payments, refunds, idempotency, webhook handling, and cash/gift card payments, GetBookingsByCreatedRange endpoint, scheduling exceptional hours validation, referral code registration, JWT revocation via JTI logout, and GDPR compliance (export handler cache states, anonymize_user child table scrubbing, export_all_user_data 16-section export, AnonymizeStaleGuestAccounts field scrubbing). +**584/587 tests passing** (3 skipped) across 12+ packages. Comprehensive coverage of online payments (deposit, full, partial, balance), saved card operations, tip payments, terminal payments, refunds, idempotency, webhook handling, and cash/gift card payments, GetBookingsByCreatedRange endpoint, scheduling exceptional hours validation, referral code registration, JWT revocation via JTI logout, GDPR compliance (export handler cache states, anonymize_user child table scrubbing, export_all_user_data 16-section export, AnonymizeStaleGuestAccounts field scrubbing), and financial data retention (7-year expiry, 1-year post-anonymization buffer, aggregation correctness, refund handling, idempotency). - `handlers/auth` — Authentication (login, register, referral code validation, refresh, verification) - `handlers/bookings` — User booking flow, guest bookings, reservations, edit requests, discounts, closing hours validation, active booking limits, GetBookingsByCreatedRange - `handlers/payments` — Square payments (terminal, online, refunds, tips, saved cards) - `internal/square` — Square client dev mock tests - `handlers/admin` — Admin bookings, today view, users, services, GetBookingsByCreatedRange -- `handlers/scheduling` — Working hours, exceptional groups, available hours, time blockers, exceptional hours validation +- `handlers/scheduling` — Working hours, exceptional groups, available hours, time blockers, exceptional hours validation, financial data retention & aggregation - `handlers/services` — Service eligibility - `handlers/user` — User profile, guest creation, loyalty - `handlers/portfolio` — Image upload, listing, tags, filters