fix: adminnotify observability — money-critical rows sort first, flood-cap suppression surfaced to operator, stale coordination doc fixed
- notifications priority ordering: money-critical reasons (webhooks, sweeps, refunds, gift-card, manual-refund failures) above routine - admin notifications page exposes the flood-cap suppressed count - adminnotify.go contract doc: removed stale 2FA reissue-fail site, current insert-site list Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)
This commit is contained in:
@@ -3,6 +3,7 @@ package notifications
|
||||
import (
|
||||
"context"
|
||||
"crussell/db"
|
||||
"crussell/internal/adminnotify"
|
||||
"crussell/internal/validators"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
@@ -27,8 +28,24 @@ type AdminNotification struct {
|
||||
UserID *string `json:"user_id,omitempty"`
|
||||
UserName *string `json:"user_name,omitempty"`
|
||||
BookingStartTime *time.Time `json:"booking_start_time,omitempty"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Money-critical event detail (amount / square id / description): populated
|
||||
// by the 'critical_payment_log' / 'refund_failed' insert sites when they
|
||||
// adopt the admin_notifications event-detail columns (see the column
|
||||
// contract on adminnotify.go) so the operator can see WHAT happened without
|
||||
// opening the CRITICAL logs.
|
||||
Amount *float64 `json:"amount,omitempty"`
|
||||
SquareID *string `json:"square_id,omitempty"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// AdminNotificationSuppression is one flood-cap suppression counter for a
|
||||
// capped reason whose unacknowledged queue is still at the cap.
|
||||
type AdminNotificationSuppression struct {
|
||||
Reason string `json:"reason"`
|
||||
SuppressedCount int `json:"suppressed_count"`
|
||||
LastSuppressedAt time.Time `json:"last_suppressed_at"`
|
||||
}
|
||||
|
||||
type AdminNotificationListResponse struct {
|
||||
@@ -37,6 +54,13 @@ type AdminNotificationListResponse struct {
|
||||
PerPage int `json:"per_page"`
|
||||
Total int `json:"total"`
|
||||
NextCursor *string `json:"next_cursor,omitempty"`
|
||||
// Suppressed is how many money-critical alerts were dropped by the flood cap
|
||||
// (adminnotify.MaxUnacknowledgedCriticalLogs) while each reason's
|
||||
// unacknowledged queue stayed at the cap — the "suppressed this cycle"
|
||||
// count. It resets once the operator works the queue down. SuppressedDetails
|
||||
// carries the per-reason breakdown.
|
||||
Suppressed int `json:"suppressed"`
|
||||
SuppressedDetails []AdminNotificationSuppression `json:"suppressed_details,omitempty"`
|
||||
}
|
||||
|
||||
// parseCursor splits a "createdAt|id" cursor string into its components.
|
||||
@@ -47,6 +71,13 @@ type AdminNotificationListResponse struct {
|
||||
// cursor, per_page — pagination (cursor-based)
|
||||
// include_acknowledged — if "true", returns all notifications sorted newest-first.
|
||||
// Default (false/omitted): only unacknowledged, sorted by priority then oldest-first.
|
||||
// Money-critical reasons ('critical_payment_log', 'refund_failed',
|
||||
// 'refresh_token_reuse', 'gift_card_purchased_for_friend') sort ABOVE routine
|
||||
// notifications so the operator's only pager surfaces money/security events first.
|
||||
// reason — filter to a single reason.
|
||||
// The response additionally carries the flood-cap "suppressed this cycle" count
|
||||
// (suppressed / suppressed_details) for reasons whose unacknowledged queue is
|
||||
// still at adminnotify.MaxUnacknowledgedCriticalLogs.
|
||||
func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse query params
|
||||
perPage := 20
|
||||
@@ -63,6 +94,7 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
SELECT an.id, an.reason, an.booking_id, an.user_id,
|
||||
u.n_first_name || ' ' || u.n_last_name AS user_name,
|
||||
b.start_time AS booking_start_time,
|
||||
an.amount, an.square_id, an.description,
|
||||
an.acknowledged_at, an.created_at
|
||||
FROM admin_notifications an
|
||||
LEFT JOIN users u ON an.user_id = u.id
|
||||
@@ -108,16 +140,24 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
baseQuery += " ORDER BY an.created_at DESC, an.id DESC"
|
||||
} else {
|
||||
baseQuery += ` ORDER BY CASE an.reason
|
||||
WHEN 'pending_booking' THEN 1
|
||||
WHEN 'cancelled_booking' THEN 2
|
||||
WHEN 'late_cancellation' THEN 3
|
||||
WHEN 'deposit_paid' THEN 4
|
||||
WHEN 'affiliate_claim' THEN 5
|
||||
WHEN 'edit_requested' THEN 6
|
||||
WHEN 'new_booking' THEN 7
|
||||
WHEN '1_month_no_pay' THEN 8
|
||||
WHEN '1_week_no_pay' THEN 9
|
||||
ELSE 10
|
||||
WHEN 'critical_payment_log' THEN 1
|
||||
WHEN 'refund_failed' THEN 2
|
||||
WHEN 'refresh_token_reuse' THEN 3
|
||||
WHEN 'gift_card_purchased_for_friend' THEN 4
|
||||
WHEN 'pending_booking' THEN 5
|
||||
WHEN 'cancelled_booking' THEN 6
|
||||
WHEN 'late_cancellation' THEN 7
|
||||
WHEN 'deposit_paid' THEN 8
|
||||
WHEN 'affiliate_claim' THEN 9
|
||||
WHEN 'edit_requested' THEN 10
|
||||
WHEN 'new_booking' THEN 11
|
||||
WHEN '1_month_no_pay' THEN 12
|
||||
WHEN '1_week_no_pay' THEN 13
|
||||
WHEN 'rescheduled_booking' THEN 14
|
||||
WHEN 'edit_request' THEN 15
|
||||
WHEN 'deposit_not_paid_by_deadline' THEN 16
|
||||
WHEN 'default_hours_changed' THEN 17
|
||||
ELSE 18
|
||||
END, an.created_at ASC, an.id ASC`
|
||||
}
|
||||
baseQuery += fmt.Sprintf(" LIMIT $%d", param)
|
||||
@@ -149,6 +189,9 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
var userID sql.NullString
|
||||
var userName sql.NullString
|
||||
var bookingStartTime sql.NullTime
|
||||
var amount sql.NullFloat64
|
||||
var squareID sql.NullString
|
||||
var description sql.NullString
|
||||
var acknowledgedAt sql.NullTime
|
||||
|
||||
err := rows.Scan(
|
||||
@@ -158,6 +201,9 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
&userID,
|
||||
&userName,
|
||||
&bookingStartTime,
|
||||
&amount,
|
||||
&squareID,
|
||||
&description,
|
||||
&acknowledgedAt,
|
||||
&n.CreatedAt,
|
||||
)
|
||||
@@ -179,6 +225,15 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
if bookingStartTime.Valid {
|
||||
n.BookingStartTime = &bookingStartTime.Time
|
||||
}
|
||||
if amount.Valid {
|
||||
n.Amount = &amount.Float64
|
||||
}
|
||||
if squareID.Valid {
|
||||
n.SquareID = &squareID.String
|
||||
}
|
||||
if description.Valid {
|
||||
n.Description = &description.String
|
||||
}
|
||||
if acknowledgedAt.Valid {
|
||||
n.AcknowledgedAt = &acknowledgedAt.Time
|
||||
}
|
||||
@@ -206,6 +261,20 @@ func GetNotifications(w http.ResponseWriter, r *http.Request) {
|
||||
NextCursor: nextCursor,
|
||||
}
|
||||
|
||||
if suppressions, err := adminnotify.ActiveSuppressions(r.Context(), db.Conn); err != nil {
|
||||
log.Printf("Failed to fetch flood-cap suppressions: %v", err)
|
||||
} else {
|
||||
resp.SuppressedDetails = make([]AdminNotificationSuppression, 0, len(suppressions))
|
||||
for _, s := range suppressions {
|
||||
resp.Suppressed += s.SuppressedCount
|
||||
resp.SuppressedDetails = append(resp.SuppressedDetails, AdminNotificationSuppression{
|
||||
Reason: s.Reason,
|
||||
SuppressedCount: s.SuppressedCount,
|
||||
LastSuppressedAt: s.LastSuppressedAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(resp); err != nil {
|
||||
log.Printf("Failed to encode response: %v", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"crussell/db"
|
||||
"crussell/internal/adminnotify"
|
||||
"crussell/mw"
|
||||
"crussell/testutils"
|
||||
|
||||
@@ -691,3 +692,196 @@ func TestAcknowledgeNotification_BeginError(t *testing.T) {
|
||||
t.Errorf("expected 500, got %d. body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Money-critical priority ordering (FIX 2)
|
||||
// =============================================================================
|
||||
|
||||
// TestNotifications_Priority_MoneyCriticalFirst pins FIX 2: money-critical
|
||||
// reasons (webhook/sweep 'critical_payment_log', 'refund_failed',
|
||||
// 'refresh_token_reuse', gift-card events) must sort ABOVE routine
|
||||
// notifications in the unacknowledged feed so the operator's only pager
|
||||
// surfaces money/security events first. The money rows are inserted LAST to
|
||||
// prove the priority CASE — not insertion order — drives the sort.
|
||||
func TestNotifications_Priority_MoneyCriticalFirst(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, tx := testutils.SetupTestTx(t)
|
||||
userID := createTestUser(t, ctx, tx)
|
||||
|
||||
moneyReasons := []string{
|
||||
"critical_payment_log",
|
||||
"refund_failed",
|
||||
"refresh_token_reuse",
|
||||
"gift_card_purchased_for_friend",
|
||||
}
|
||||
routineReasons := []string{
|
||||
"pending_booking",
|
||||
"cancelled_booking",
|
||||
"late_cancellation",
|
||||
"deposit_paid",
|
||||
"affiliate_claim",
|
||||
"edit_requested",
|
||||
"new_booking",
|
||||
"1_month_no_pay",
|
||||
"1_week_no_pay",
|
||||
"rescheduled_booking",
|
||||
"edit_request",
|
||||
"deposit_not_paid_by_deadline",
|
||||
"default_hours_changed",
|
||||
}
|
||||
moneySet := make(map[string]bool, len(moneyReasons))
|
||||
for _, r := range moneyReasons {
|
||||
moneySet[r] = true
|
||||
}
|
||||
|
||||
for _, reason := range routineReasons {
|
||||
createNotification(t, ctx, tx, reason, userID, false)
|
||||
}
|
||||
for _, reason := range moneyReasons {
|
||||
createNotification(t, ctx, tx, reason, userID, false)
|
||||
}
|
||||
|
||||
handler := http.HandlerFunc(GetNotifications)
|
||||
w := makeExtendedAdminRequest(handler, "GET", "/api/admin/notifications", nil, ctx)
|
||||
|
||||
var resp AdminNotificationListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Notifications) != len(moneyReasons)+len(routineReasons) {
|
||||
t.Fatalf("expected %d notifications, got %d", len(moneyReasons)+len(routineReasons), len(resp.Notifications))
|
||||
}
|
||||
|
||||
// Every money-critical reason must appear strictly before the first routine
|
||||
// reason; each money reason must be present.
|
||||
lastMoney := -1
|
||||
firstRoutine := len(resp.Notifications)
|
||||
for i, n := range resp.Notifications {
|
||||
if moneySet[n.Reason] {
|
||||
lastMoney = i
|
||||
} else if firstRoutine == len(resp.Notifications) {
|
||||
firstRoutine = i
|
||||
}
|
||||
}
|
||||
if lastMoney == -1 {
|
||||
t.Fatal("expected at least one money-critical reason in the response")
|
||||
}
|
||||
if lastMoney > firstRoutine {
|
||||
t.Errorf("money-critical reasons must sort above routine notifications: last money position %d, first routine position %d", lastMoney, firstRoutine)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Flood-cap suppression count in the response (FIX 3a)
|
||||
// =============================================================================
|
||||
|
||||
// TestNotifications_Response_SuppressedCount pins FIX 3a: when the flood cap
|
||||
// has suppressed alerts, GET /api/admin/notifications exposes the per-reason
|
||||
// "suppressed this cycle" count, and acknowledging the queue back below the cap
|
||||
// resets it to zero.
|
||||
func TestNotifications_Response_SuppressedCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, tx := testutils.SetupTestTx(t)
|
||||
userID := createTestUser(t, ctx, tx)
|
||||
handler := http.HandlerFunc(GetNotifications)
|
||||
|
||||
// No suppressions yet → the response reports zero.
|
||||
w := makeExtendedAdminRequest(handler, "GET", "/api/admin/notifications", nil, ctx)
|
||||
var resp AdminNotificationListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal: %v", err)
|
||||
}
|
||||
if resp.Suppressed != 0 {
|
||||
t.Errorf("expected suppressed 0 before any cap hit, got %d", resp.Suppressed)
|
||||
}
|
||||
if len(resp.SuppressedDetails) != 0 {
|
||||
t.Errorf("expected no suppressed_details before any cap hit, got %d", len(resp.SuppressedDetails))
|
||||
}
|
||||
|
||||
// Fill the critical_payment_log queue to the cap, then hit the cap twice the
|
||||
// way an insert site would (pre-check records each suppression).
|
||||
for i := 0; i < adminnotify.MaxUnacknowledgedCriticalLogs; i++ {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO admin_notifications (reason, user_id, created_at)
|
||||
VALUES ('critical_payment_log', $1, NOW())
|
||||
`, userID); err != nil {
|
||||
t.Fatalf("failed to fill the queue to the cap: %v", err)
|
||||
}
|
||||
}
|
||||
if !adminnotify.CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") {
|
||||
t.Fatal("expected the queue to be at the cap")
|
||||
}
|
||||
if !adminnotify.CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") {
|
||||
t.Fatal("expected the queue to stay at the cap")
|
||||
}
|
||||
|
||||
w = makeExtendedAdminRequest(handler, "GET", "/api/admin/notifications", nil, ctx)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal: %v", err)
|
||||
}
|
||||
if resp.Suppressed != 2 {
|
||||
t.Errorf("expected suppressed 2, got %d", resp.Suppressed)
|
||||
}
|
||||
if len(resp.SuppressedDetails) != 1 {
|
||||
t.Fatalf("expected 1 suppressed_details entry, got %d", len(resp.SuppressedDetails))
|
||||
}
|
||||
if resp.SuppressedDetails[0].Reason != "critical_payment_log" {
|
||||
t.Errorf("expected suppressed_details reason critical_payment_log, got %s", resp.SuppressedDetails[0].Reason)
|
||||
}
|
||||
if resp.SuppressedDetails[0].SuppressedCount != 2 {
|
||||
t.Errorf("expected suppressed_details count 2, got %d", resp.SuppressedDetails[0].SuppressedCount)
|
||||
}
|
||||
|
||||
// Acknowledging the queue below the cap resets the visible count.
|
||||
if _, err := tx.Exec(ctx, "UPDATE admin_notifications SET acknowledged_at = NOW() WHERE reason = 'critical_payment_log'"); err != nil {
|
||||
t.Fatalf("failed to acknowledge the queue: %v", err)
|
||||
}
|
||||
w = makeExtendedAdminRequest(handler, "GET", "/api/admin/notifications", nil, ctx)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal: %v", err)
|
||||
}
|
||||
if resp.Suppressed != 0 {
|
||||
t.Errorf("expected suppressed 0 after the queue was worked down, got %d", resp.Suppressed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifications_Response_CarriesEventDetail pins FIX 3b: the GET endpoint
|
||||
// surfaces the money-critical event detail columns (amount, square_id,
|
||||
// description) when the insert site has populated them.
|
||||
func TestNotifications_Response_CarriesEventDetail(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, tx := testutils.SetupTestTx(t)
|
||||
|
||||
var notificationID string
|
||||
err := tx.QueryRow(ctx, `
|
||||
INSERT INTO admin_notifications (reason, amount, square_id, description)
|
||||
VALUES ('critical_payment_log', 42.50, 'sq_dispute_123', 'Dispute received for charge 42.50')
|
||||
RETURNING id
|
||||
`).Scan(¬ificationID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create notification with event detail: %v", err)
|
||||
}
|
||||
|
||||
handler := http.HandlerFunc(GetNotifications)
|
||||
w := makeExtendedAdminRequest(handler, "GET", "/api/admin/notifications", nil, ctx)
|
||||
|
||||
var resp AdminNotificationListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Notifications) != 1 {
|
||||
t.Fatalf("expected 1 notification, got %d", len(resp.Notifications))
|
||||
}
|
||||
n := resp.Notifications[0]
|
||||
if n.Amount == nil || *n.Amount != 42.50 {
|
||||
t.Errorf("expected amount 42.50, got %v", n.Amount)
|
||||
}
|
||||
if n.SquareID == nil || *n.SquareID != "sq_dispute_123" {
|
||||
t.Errorf("expected square_id sq_dispute_123, got %v", n.SquareID)
|
||||
}
|
||||
if n.Description == nil || *n.Description != "Dispute received for charge 42.50" {
|
||||
t.Errorf("expected description populated, got %v", n.Description)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,39 +2,47 @@
|
||||
// notification centre (admin_notifications). That table is the single
|
||||
// operator's ONLY pager for money events, so every site that inserts an
|
||||
// operator-facing alert must bound its unacknowledged queue — a hostile flood
|
||||
// (attacker-registered accounts triggering refresh_token_reuse / reissue-fail /
|
||||
// webhook alerts) must not be able to bury the notification centre under rows
|
||||
// the operator can never work through.
|
||||
// (attacker-registered accounts triggering refresh_token_reuse / webhook
|
||||
// alerts) must not be able to bury the notification centre under rows the
|
||||
// operator can never work through.
|
||||
//
|
||||
// Coordination contract (Round 2 Loop B finding 1): the cap is applied
|
||||
// atomically at every insert site in this codebase:
|
||||
//
|
||||
// - handlers/payments/twofa.go — the reissue-fail alert (per-issue capped,
|
||||
// see finding 2; NOT globally capped).
|
||||
// - auth/jwt.go VerifyRefreshToken — the 'refresh_token_reuse' alert.
|
||||
// - handlers/webhooks/square.go — the three critical_payment_log inserts
|
||||
// - handlers/webhooks/square.go — the 'critical_payment_log' inserts
|
||||
// (dispute, booking, unknown-event, orphan-replay).
|
||||
// - handlers/user/account.go InsertSquareErasureCriticalNotification.
|
||||
// - handlers/payments/sweep.go insertCriticalPaymentNotification.
|
||||
// - handlers/scheduling/time-blockers.go InsertSquareCleanupCriticalNotification
|
||||
// and the deposit-deadline cleanup insert.
|
||||
// - internal/jobs/cleanup.go ScanCriticalPaymentLogs.
|
||||
// - handlers/scheduling/scheduled-cleanup.go — the '1_week_no_pay',
|
||||
// '1_month_no_pay' and 'default_hours_changed' inserts.
|
||||
// - handlers/bookings/bookings.go — the 'new_booking' / 'pending_booking' /
|
||||
// 'cancelled_booking' inserts.
|
||||
// - handlers/bookings/manage.go — the 'cancelled_booking' / 'edit_requested'
|
||||
// inserts.
|
||||
// - handlers/payments/refunds.go — the 'refund_failed' insert.
|
||||
//
|
||||
// Sites owned by OTHER agents (coordination notes):
|
||||
//
|
||||
// - handlers/payments/sweep.go insertCriticalPaymentNotification — NOW capped
|
||||
// (same atomic fold + pre-check as every other site).
|
||||
// - handlers/scheduling/time-blockers.go insertSquareCleanupCriticalNotification
|
||||
// and internal/jobs/cleanup.go ScanCriticalPaymentLogs — both NOW capped
|
||||
// (same atomic fold + pre-check as every other site).
|
||||
// - main.go has NO admin_notifications insert sites (it only mounts the
|
||||
// notification read/ack routes), so nothing to cap there.
|
||||
// main.go has NO admin_notifications insert sites (it only mounts the
|
||||
// notification read/ack routes), so nothing to cap there.
|
||||
//
|
||||
// Every site folds the cap INTO its INSERT (a conditional
|
||||
// `INSERT ... SELECT ... WHERE (SELECT COUNT(*) ...) < $cap`) so the
|
||||
// count-then-insert is ATOMIC — closing the TOCTOU where two concurrent
|
||||
// inserts both read a below-cap count and overshoot together (finding 2).
|
||||
// Every site ALSO pre-checks CriticalLogsCapExceeded to log the suppression;
|
||||
// that pre-check is the single choke point that records the suppression in
|
||||
// admin_notification_suppressions, so the operator-facing "suppressed this
|
||||
// cycle" count on GET /api/admin/notifications stays accurate without any
|
||||
// insert site having to change.
|
||||
package adminnotify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"crussell/db"
|
||||
)
|
||||
@@ -58,6 +66,11 @@ const MaxUnacknowledgedCriticalLogs = 100
|
||||
// a money alert is never dropped because the count query failed (the insert's
|
||||
// own atomic cap condition below still guards the row in that case — the
|
||||
// pre-check only decides whether to log a suppression).
|
||||
//
|
||||
// At the cap the suppression is ALSO recorded in admin_notification_suppressions
|
||||
// (recordSuppression) so the admin notifications page can surface how many
|
||||
// alerts were dropped. The record write is best-effort too — a failure only
|
||||
// loses the counter, never the boolean decision.
|
||||
func CriticalLogsCapExceeded(ctx context.Context, q db.Querier, reason string) bool {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
@@ -68,5 +81,66 @@ func CriticalLogsCapExceeded(ctx context.Context, q db.Querier, reason string) b
|
||||
log.Printf("adminnotify: failed to count unacknowledged %s admin notifications: %v", reason, err)
|
||||
return false
|
||||
}
|
||||
return n >= MaxUnacknowledgedCriticalLogs
|
||||
if n >= MaxUnacknowledgedCriticalLogs {
|
||||
recordSuppression(ctx, q, reason)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// recordSuppression upserts the per-reason flood-cap suppression counter
|
||||
// (admin_notification_suppressions). Every insert site funnels its pre-check
|
||||
// through CriticalLogsCapExceeded, so a suppression is recorded exactly once
|
||||
// per dropped alert, and a running suppressed_count accumulates while the
|
||||
// reason's unacknowledged queue stays at the cap. Best-effort: a failure is
|
||||
// logged, never propagated — the alert is already being dropped at the cap and
|
||||
// the operator-facing counter is informational, not load-bearing.
|
||||
func recordSuppression(ctx context.Context, q db.Querier, reason string) {
|
||||
if _, err := q.Exec(ctx, `
|
||||
INSERT INTO admin_notification_suppressions (reason, suppressed_count, first_suppressed_at, last_suppressed_at)
|
||||
VALUES ($1::admin_notification_reason, 1, NOW(), NOW())
|
||||
ON CONFLICT (reason) DO UPDATE SET
|
||||
suppressed_count = admin_notification_suppressions.suppressed_count + 1,
|
||||
last_suppressed_at = NOW()
|
||||
`, reason); err != nil {
|
||||
log.Printf("adminnotify: failed to record flood-cap suppression for %s: %v", reason, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Suppression is the per-reason flood-cap suppression counter surfaced to the
|
||||
// operator: how many alerts were dropped for that reason while its
|
||||
// unacknowledged queue sat at MaxUnacknowledgedCriticalLogs.
|
||||
type Suppression struct {
|
||||
Reason string
|
||||
SuppressedCount int
|
||||
LastSuppressedAt time.Time
|
||||
}
|
||||
|
||||
// ActiveSuppressions returns the per-reason flood-cap suppressions whose
|
||||
// underlying unacknowledged queue is STILL at the cap — i.e. the operator has
|
||||
// not yet worked it down ("this cycle"). The admin notifications handler sums
|
||||
// these into the response's suppressed count; the count naturally resets once
|
||||
// the operator acknowledges the queue back below the cap. Best-effort: the
|
||||
// caller must treat a non-nil error as "no suppression data available".
|
||||
func ActiveSuppressions(ctx context.Context, q db.Querier) ([]Suppression, error) {
|
||||
rows, err := q.Query(ctx, `
|
||||
SELECT s.reason, s.suppressed_count, s.last_suppressed_at
|
||||
FROM admin_notification_suppressions s
|
||||
WHERE (SELECT COUNT(*) FROM admin_notifications an
|
||||
WHERE an.reason = s.reason AND an.acknowledged_at IS NULL) >= $1
|
||||
ORDER BY s.last_suppressed_at DESC
|
||||
`, MaxUnacknowledgedCriticalLogs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Suppression
|
||||
for rows.Next() {
|
||||
var s Suppression
|
||||
if err := rows.Scan(&s.Reason, &s.SuppressedCount, &s.LastSuppressedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ func TestCriticalLogsCapExceeded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.Conn.Exec(ctx, "DELETE FROM admin_notifications WHERE reason = 'critical_payment_log'")
|
||||
_, _ = db.Conn.Exec(ctx, "DELETE FROM admin_notification_suppressions WHERE reason = 'critical_payment_log'")
|
||||
})
|
||||
|
||||
if CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") {
|
||||
@@ -70,3 +71,72 @@ func TestCriticalLogsCapExceeded(t *testing.T) {
|
||||
t.Fatal("expected acknowledging the queue to re-arm inserts")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCriticalLogsCapExceeded_RecordsSuppression pins FIX 3a: hitting the
|
||||
// flood cap records the suppression in admin_notification_suppressions (the
|
||||
// operator-facing "suppressed this cycle" counter) with a running count, and
|
||||
// ActiveSuppressions reports it only while the underlying queue is still at the
|
||||
// cap — acknowledging the queue down resets the visible count.
|
||||
func TestCriticalLogsCapExceeded_RecordsSuppression(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.Conn.Exec(ctx, "DELETE FROM admin_notifications WHERE reason = 'critical_payment_log'")
|
||||
_, _ = db.Conn.Exec(ctx, "DELETE FROM admin_notification_suppressions WHERE reason = 'critical_payment_log'")
|
||||
})
|
||||
|
||||
// Below the cap no suppression is recorded.
|
||||
for i := 0; i < MaxUnacknowledgedCriticalLogs-1; i++ {
|
||||
if _, err := db.Conn.Exec(ctx, `
|
||||
INSERT INTO admin_notifications (reason, created_at)
|
||||
VALUES ('critical_payment_log', NOW())
|
||||
`); err != nil {
|
||||
t.Fatalf("failed to insert row %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if supps, err := ActiveSuppressions(ctx, db.Conn); err != nil {
|
||||
t.Fatalf("ActiveSuppressions failed below the cap: %v", err)
|
||||
} else if len(supps) != 0 {
|
||||
t.Fatalf("expected no suppression below the cap, got %d", len(supps))
|
||||
}
|
||||
|
||||
// Fill to the cap, then hit it twice: each suppressed alert is recorded.
|
||||
if _, err := db.Conn.Exec(ctx, `
|
||||
INSERT INTO admin_notifications (reason, created_at)
|
||||
VALUES ('critical_payment_log', NOW())
|
||||
`); err != nil {
|
||||
t.Fatalf("failed to insert the cap row: %v", err)
|
||||
}
|
||||
if !CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") {
|
||||
t.Fatal("expected the queue to be at the cap")
|
||||
}
|
||||
if !CriticalLogsCapExceeded(ctx, db.Conn, "critical_payment_log") {
|
||||
t.Fatal("expected the queue to stay at the cap")
|
||||
}
|
||||
|
||||
supps, err := ActiveSuppressions(ctx, db.Conn)
|
||||
if err != nil {
|
||||
t.Fatalf("ActiveSuppressions failed: %v", err)
|
||||
}
|
||||
if len(supps) != 1 {
|
||||
t.Fatalf("expected exactly 1 suppression record, got %d", len(supps))
|
||||
}
|
||||
if supps[0].Reason != "critical_payment_log" {
|
||||
t.Errorf("expected reason critical_payment_log, got %s", supps[0].Reason)
|
||||
}
|
||||
if supps[0].SuppressedCount != 2 {
|
||||
t.Errorf("expected suppressed_count 2, got %d", supps[0].SuppressedCount)
|
||||
}
|
||||
|
||||
// Acknowledging the queue below the cap makes the suppression inactive, so
|
||||
// the "this cycle" count resets.
|
||||
if _, err := db.Conn.Exec(ctx, "UPDATE admin_notifications SET acknowledged_at = NOW() WHERE reason = 'critical_payment_log'"); err != nil {
|
||||
t.Fatalf("failed to acknowledge the queue: %v", err)
|
||||
}
|
||||
supps, err = ActiveSuppressions(ctx, db.Conn)
|
||||
if err != nil {
|
||||
t.Fatalf("ActiveSuppressions failed after acknowledge: %v", err)
|
||||
}
|
||||
if len(supps) != 0 {
|
||||
t.Errorf("expected the suppression to reset once the queue is below the cap, got %d", len(supps))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user