Skip to content

Commit 840d730

Browse files
wrn14897wrn14897
authored
feat: adjust alert template title and body to reflect alert state (#1339)
Currently, the resolved alert will have the same title and body message as the alerting one, which is misleading Ref: HDX-2786 ## Slack ### ALERT <img width="723" height="358" alt="Screenshot 2025-11-09 at 10 02 52 PM" src="https://github.com/user-attachments/assets/b1c6f563-f095-457e-9a70-01c8149796c4" /> ### RESOLVED <img width="650" height="117" alt="Screenshot 2025-11-09 at 10 26 01 PM" src="https://github.com/user-attachments/assets/07ef1e7d-8ee5-4604-92cf-4811a0a5c811" /> ## incident.io ### ALERT <img width="1432" height="398" alt="Screenshot 2025-11-09 at 11 07 30 PM" src="https://github.com/user-attachments/assets/30e25eb3-32b2-4f51-934d-b28e75dd5cf7" /> ### RESOLVED <img width="1427" height="305" alt="Screenshot 2025-11-09 at 11 08 56 PM" src="https://github.com/user-attachments/assets/913a5b99-bb07-47ae-bec9-6b0814e4b400" />
1 parent b33db76 commit 840d730

File tree

4 files changed

+159
-24
lines changed

4 files changed

+159
-24
lines changed

‎.changeset/spotty-yaks-sniff.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/api": minor
3+
---
4+
5+
feat: adjust alert template title and body to reflect alert state

‎packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts‎

Lines changed: 133 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import {
4343
buildAlertMessageTemplateHdxLink,
4444
buildAlertMessageTemplateTitle,
4545
getDefaultExternalAction,
46+
isAlertResolved,
4647
renderAlertTemplate,
4748
translateExternalActionsToInternal,
4849
} from '@/tasks/checkAlerts/template';
@@ -229,16 +230,70 @@ describe('checkAlerts', () => {
229230
buildAlertMessageTemplateTitle({
230231
view: defaultSearchView,
231232
}),
232-
).toMatchInlineSnapshot(`"Alert for \\"My Search\\" - 10 lines found"`);
233+
).toMatchInlineSnapshot(
234+
`"🚨 Alert for \\"My Search\\" - 10 lines found"`,
235+
);
233236
expect(
234237
buildAlertMessageTemplateTitle({
235238
view: defaultChartView,
236239
}),
237240
).toMatchInlineSnapshot(
238-
`"Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
241+
`"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
239242
);
240243
});
241244

245+
it('buildAlertMessageTemplateTitle with state parameter', () => {
246+
// Test ALERT state (should have 🚨 emoji)
247+
expect(
248+
buildAlertMessageTemplateTitle({
249+
view: defaultSearchView,
250+
state: AlertState.ALERT,
251+
}),
252+
).toMatchInlineSnapshot(
253+
`"🚨 Alert for \\"My Search\\" - 10 lines found"`,
254+
);
255+
expect(
256+
buildAlertMessageTemplateTitle({
257+
view: defaultChartView,
258+
state: AlertState.ALERT,
259+
}),
260+
).toMatchInlineSnapshot(
261+
`"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
262+
);
263+
264+
// Test OK state (should have ✅ emoji)
265+
expect(
266+
buildAlertMessageTemplateTitle({
267+
view: defaultSearchView,
268+
state: AlertState.OK,
269+
}),
270+
).toMatchInlineSnapshot(
271+
`"✅ Alert for \\"My Search\\" - 10 lines found"`,
272+
);
273+
expect(
274+
buildAlertMessageTemplateTitle({
275+
view: defaultChartView,
276+
state: AlertState.OK,
277+
}),
278+
).toMatchInlineSnapshot(
279+
`"✅ Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
280+
);
281+
});
282+
283+
it('isAlertResolved', () => {
284+
// Test OK state returns true
285+
expect(isAlertResolved(AlertState.OK)).toBe(true);
286+
287+
// Test ALERT state returns false
288+
expect(isAlertResolved(AlertState.ALERT)).toBe(false);
289+
290+
// Test INSUFFICIENT_DATA state returns false
291+
expect(isAlertResolved(AlertState.INSUFFICIENT_DATA)).toBe(false);
292+
293+
// Test DISABLED state returns false
294+
expect(isAlertResolved(AlertState.DISABLED)).toBe(false);
295+
});
296+
242297
it('getDefaultExternalAction', () => {
243298
expect(
244299
getDefaultExternalAction({
@@ -372,7 +427,7 @@ describe('checkAlerts', () => {
372427
},
373428
},
374429
},
375-
title: 'Alert for "My Search" - 10 lines found',
430+
title: '🚨 Alert for "My Search" - 10 lines found',
376431
teamWebhooksById: new Map<string, typeof webhook>([
377432
[webhook._id.toString(), webhook],
378433
]),
@@ -382,12 +437,12 @@ describe('checkAlerts', () => {
382437
1,
383438
'https://hooks.slack.com/services/123',
384439
{
385-
text: 'Alert for "My Search" - 10 lines found',
440+
text: '🚨 Alert for "My Search" - 10 lines found',
386441
blocks: [
387442
{
388443
text: {
389444
text: [
390-
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
445+
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
391446
'Group: "http"',
392447
'10 lines found, expected less than 1 lines',
393448
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -432,7 +487,7 @@ describe('checkAlerts', () => {
432487
webhookName: 'My_Webhook',
433488
},
434489
},
435-
title: 'Alert for "My Search" - 10 lines found',
490+
title: '🚨 Alert for "My Search" - 10 lines found',
436491
teamWebhooksById: new Map<string, typeof webhook>([
437492
[webhook._id.toString(), webhook],
438493
]),
@@ -442,12 +497,12 @@ describe('checkAlerts', () => {
442497
1,
443498
'https://hooks.slack.com/services/123',
444499
{
445-
text: 'Alert for "My Search" - 10 lines found',
500+
text: '🚨 Alert for "My Search" - 10 lines found',
446501
blocks: [
447502
{
448503
text: {
449504
text: [
450-
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
505+
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
451506
'Group: "http"',
452507
'10 lines found, expected less than 1 lines',
453508
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -517,7 +572,7 @@ describe('checkAlerts', () => {
517572
},
518573
},
519574
},
520-
title: 'Alert for "My Search" - 10 lines found',
575+
title: '🚨 Alert for "My Search" - 10 lines found',
521576
teamWebhooksById,
522577
});
523578

@@ -541,20 +596,20 @@ describe('checkAlerts', () => {
541596
host: 'web2',
542597
},
543598
},
544-
title: 'Alert for "My Search" - 10 lines found',
599+
title: '🚨 Alert for "My Search" - 10 lines found',
545600
teamWebhooksById,
546601
});
547602

548603
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2);
549604
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
550605
'https://hooks.slack.com/services/123',
551606
{
552-
text: 'Alert for "My Search" - 10 lines found',
607+
text: '🚨 Alert for "My Search" - 10 lines found',
553608
blocks: [
554609
{
555610
text: {
556611
text: [
557-
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
612+
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
558613
'Group: "http"',
559614
'10 lines found, expected less than 1 lines',
560615
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -578,12 +633,12 @@ describe('checkAlerts', () => {
578633
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
579634
'https://hooks.slack.com/services/456',
580635
{
581-
text: 'Alert for "My Search" - 10 lines found',
636+
text: '🚨 Alert for "My Search" - 10 lines found',
582637
blocks: [
583638
{
584639
text: {
585640
text: [
586-
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
641+
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
587642
'Group: "http"',
588643
'10 lines found, expected less than 1 lines',
589644
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -605,6 +660,63 @@ describe('checkAlerts', () => {
605660
},
606661
);
607662
});
663+
664+
it('renderAlertTemplate - resolved alert with simplified message', async () => {
665+
const team = await createTeam({ name: 'My Team' });
666+
const webhook = await new Webhook({
667+
team: team._id,
668+
service: 'slack',
669+
url: 'https://hooks.slack.com/services/123',
670+
name: 'My_Webhook',
671+
}).save();
672+
673+
await renderAlertTemplate({
674+
alertProvider,
675+
clickhouseClient: {} as any,
676+
metadata: {} as any,
677+
state: AlertState.OK, // Resolved state
678+
template: '@webhook-My_Webhook',
679+
view: {
680+
...defaultSearchView,
681+
alert: {
682+
...defaultSearchView.alert,
683+
channel: {
684+
type: null, // using template instead
685+
},
686+
},
687+
},
688+
title: '✅ Alert for "My Search" - 10 lines found',
689+
teamWebhooksById: new Map<string, typeof webhook>([
690+
[webhook._id.toString(), webhook],
691+
]),
692+
});
693+
694+
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
695+
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
696+
'https://hooks.slack.com/services/123',
697+
{
698+
text: '✅ Alert for "My Search" - 10 lines found',
699+
blocks: [
700+
{
701+
text: {
702+
text: expect.stringContaining('The alert has been resolved'),
703+
type: 'mrkdwn',
704+
},
705+
type: 'section',
706+
},
707+
],
708+
},
709+
);
710+
711+
// Verify the message includes the time range but not detailed logs
712+
const callArgs = (slack.postMessageToWebhook as any).mock.calls[0][1];
713+
const messageText = callArgs.blocks[0].text.text;
714+
expect(messageText).toContain('The alert has been resolved');
715+
expect(messageText).toContain('Time Range (UTC):');
716+
expect(messageText).toContain('Group: "http"');
717+
// Should NOT contain detailed log data
718+
expect(messageText).not.toContain('lines found, expected');
719+
});
608720
});
609721

610722
describe('processAlert', () => {
@@ -915,7 +1027,7 @@ describe('checkAlerts', () => {
9151027
1,
9161028
'https://hooks.slack.com/services/123',
9171029
{
918-
text: 'Alert for "My Search" - 3 lines found',
1030+
text: '🚨 Alert for "My Search" - 3 lines found',
9191031
blocks: [
9201032
{
9211033
text: expect.any(Object),
@@ -928,7 +1040,7 @@ describe('checkAlerts', () => {
9281040
2,
9291041
'https://hooks.slack.com/services/123',
9301042
{
931-
text: 'Alert for "My Search" - 1 lines found',
1043+
text: '🚨 Alert for "My Search" - 1 lines found',
9321044
blocks: [
9331045
{
9341046
text: expect.any(Object),
@@ -1089,12 +1201,12 @@ describe('checkAlerts', () => {
10891201
1,
10901202
'https://hooks.slack.com/services/123',
10911203
{
1092-
text: 'Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
1204+
text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
10931205
blocks: [
10941206
{
10951207
text: {
10961208
text: [
1097-
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
1209+
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
10981210
'',
10991211
'3 exceeds 1',
11001212
'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',
@@ -1281,7 +1393,7 @@ describe('checkAlerts', () => {
12811393
expect(fetchMock).toHaveBeenCalledWith('https://webhook.site/123', {
12821394
method: 'POST',
12831395
body: JSON.stringify({
1284-
text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
1396+
text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
12851397
}),
12861398
headers: {
12871399
'Content-Type': 'application/json',
@@ -2087,12 +2199,12 @@ describe('checkAlerts', () => {
20872199
1,
20882200
'https://hooks.slack.com/services/123',
20892201
{
2090-
text: 'Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
2202+
text: '🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
20912203
blocks: [
20922204
{
20932205
text: {
20942206
text: [
2095-
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
2207+
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
20962208
'',
20972209
'6.25 exceeds 1',
20982210
'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',

‎packages/api/src/tasks/checkAlerts/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ const fireChannelEvent = async ({
147147
title: buildAlertMessageTemplateTitle({
148148
template: alert.name,
149149
view: templateView,
150+
state,
150151
}),
151152
template: alert.message,
152153
view: templateView,

‎packages/api/src/tasks/checkAlerts/template.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ interface Message {
8383
eventId: string;
8484
}
8585

86+
export const isAlertResolved = (state?: AlertState): boolean => {
87+
return state === AlertState.OK;
88+
};
89+
8690
export const notifyChannel = async ({
8791
channel,
8892
message,
@@ -304,20 +308,27 @@ export const buildAlertMessageTemplateHdxLink = (
304308
export const buildAlertMessageTemplateTitle = ({
305309
template,
306310
view,
311+
state,
307312
}: {
308313
template?: string | null;
309314
view: AlertMessageTemplateDefaultView;
315+
state?: AlertState;
310316
}) => {
311317
const { alert, dashboard, savedSearch, value } = view;
312318
const handlebars = createHandlebarsWithHelpers();
319+
320+
// Add emoji prefix based on alert state
321+
const emoji = isAlertResolved(state) ? '✅ ' : '🚨 ';
322+
313323
if (alert.source === AlertSource.SAVED_SEARCH) {
314324
if (savedSearch == null) {
315325
throw new Error(`Source is ${alert.source} but savedSearch is null`);
316326
}
317327
// TODO: using template engine to render the title
318-
return template
328+
const baseTitle = template
319329
? handlebars.compile(template)(view)
320330
: `Alert for "${savedSearch.name}" - ${value} lines found`;
331+
return `${emoji}${baseTitle}`;
321332
} else if (alert.source === AlertSource.TILE) {
322333
if (dashboard == null) {
323334
throw new Error(`Source is ${alert.source} but dashboard is null`);
@@ -328,7 +339,7 @@ export const buildAlertMessageTemplateTitle = ({
328339
`Tile with id ${alert.tileId} not found in dashboard ${dashboard.name}`,
329340
);
330341
}
331-
return template
342+
const baseTitle = template
332343
? handlebars.compile(template)(view)
333344
: `Alert for "${tile.config.name}" in "${dashboard.name}" - ${value} ${
334345
doesExceedThreshold(alert.thresholdType, alert.threshold, value)
@@ -339,6 +350,7 @@ export const buildAlertMessageTemplateTitle = ({
339350
? 'falls below'
340351
: 'exceeds'
341352
} ${alert.threshold}`;
353+
return `${emoji}${baseTitle}`;
342354
}
343355

344356
throw new Error(`Unsupported alert source: ${(alert as any).source}`);
@@ -527,9 +539,14 @@ export const renderAlertTemplate = async ({
527539
})})`;
528540
let rawTemplateBody;
529541

542+
// For resolved alerts, use a simple message instead of fetching data
543+
if (isAlertResolved(state)) {
544+
rawTemplateBody = `${group ? `Group: "${group}" - ` : ''}The alert has been resolved.\n${timeRangeMessage}
545+
${targetTemplate}`;
546+
}
530547
// TODO: support advanced routing with template engine
531548
// users should be able to use '@' syntax to trigger alerts
532-
if (alert.source === AlertSource.SAVED_SEARCH) {
549+
else if (alert.source === AlertSource.SAVED_SEARCH) {
533550
if (savedSearch == null) {
534551
throw new Error(`Source is ${alert.source} but savedSearch is null`);
535552
}

0 commit comments

Comments
 (0)