Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
prometheusAlerts+:: {
new(this): {
groups+: [
{
name: 'wildfly',
name: 'WildflyAlerts',
rules: [
{
alert: 'HighPercentageOfErrorResponses',
expr: |||
sum by (job, instance, server) (increase(wildfly_undertow_error_count_total{}[5m]) / increase(wildfly_undertow_request_count_total{}[5m])) * 100 > %(alertsErrorRequestErrorRate)s
||| % $._config,
||| % this.config,
'for': '5m',
labels: {
severity: 'critical',
Expand All @@ -17,23 +17,23 @@
summary: 'Large percentage of requests are resulting in 5XX responses.',
description: |||
The percentage of error responses is {{ printf "%%.2f" $value }} on {{ $labels.instance }} - {{ $labels.server }} which is higher than {{%(alertsErrorRequestErrorRate)s }}.
||| % $._config,
||| % this.config,
},
},
{
alert: 'HighNumberOfRejectedSessionsForDeployment',
alert: 'HighRejectedSessionsForDeployment',
expr: |||
sum by (deployment, instance, job) (increase(wildfly_undertow_rejected_sessions_total{}[5m])) > %(alertsErrorRejectedSessions)s
||| % $._config,
||| % this.config,
'for': '5m',
labels: {
severity: 'critical',
},
annotations: {
summary: 'Large number of sessions are being rejected for a deployment.',
description: |||
Deployemnt {{ $labels.deployment }} on {{ $labels.instance }} is exceeding the threshold for rejected sessions {{ printf "%%.0f" $value }} is higher than %(alertsErrorRejectedSessions)s.
||| % $._config,
Deployment {{ $labels.deployment }} on {{ $labels.instance }} is exceeding the threshold for rejected sessions {{ printf "%%.0f" $value }} is higher than %(alertsErrorRejectedSessions)s.
||| % this.config,
},
},
],
Expand Down
42 changes: 30 additions & 12 deletions wildfly-mixin/config.libsonnet
Original file line number Diff line number Diff line change
@@ -1,17 +1,35 @@
{
_config+:: {
enableMultiCluster: false,
multiclusterSelector: 'job=~"$job"',
wildflySelector: if self.enableMultiCluster then 'job=~"$job", cluster=~"$cluster"' else 'job=~"$job"',
dashboardTags: ['wildfly-mixin'],
dashboardPeriod: 'now-1h',
dashboardTimezone: 'default',
dashboardRefresh: '1m',
local this = self,
enableMultiCluster: false,
filteringSelector: '', // set to apply static filters to all queries and alerts, i.e. job="integrations/wildfly"
groupLabels: ['job', 'cluster'],
logLabels: ['job', 'cluster', 'instance'],
instanceLabels: ['instance'],

// alerts thresholds
alertsErrorRequestErrorRate: '30',
alertsErrorRejectedSessions: '20',
dashboardTags: [self.uid],
uid: 'wildfly',
dashboardNamePrefix: 'Wildfly',
dashboardPeriod: 'now-1h',
dashboardTimezone: 'default',
dashboardRefresh: '1m',
metricsSource: 'prometheus', // metrics source for signals

enableLokiLogs: true,
// Logging configuration
enableLokiLogs: true,
extraLogLabels: ['level', 'severity'], // Required by logs-lib
logsVolumeGroupBy: 'level',
showLogsVolume: true,

// alerts thresholds
alertsErrorRequestErrorRate: '30',
alertsErrorRejectedSessions: '20',

// Signals configuration
signals+: {
requests: (import './signals/requests.libsonnet')(this),
network: (import './signals/network.libsonnet')(this),
connections: (import './signals/connections.libsonnet')(this),
sessions: (import './signals/sessions.libsonnet')(this),
transactions: (import './signals/transactions.libsonnet')(this),
},
}
115 changes: 115 additions & 0 deletions wildfly-mixin/dashboards.libsonnet
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
local g = import './g.libsonnet';
local commonlib = import 'common-lib/common/main.libsonnet';
local logslib = import 'logs-lib/logs/main.libsonnet';
{
local root = self,
new(this)::

local links = this.grafana.links;
local tags = this.config.dashboardTags;
local uid = g.util.string.slugify(this.config.uid);
local vars = this.grafana.variables;
local annotations = this.grafana.annotations;
local refresh = this.config.dashboardRefresh;
local period = this.config.dashboardPeriod;
local timezone = this.config.dashboardTimezone;
local extraLogLabels = this.config.extraLogLabels;
{

'wildfly-overview.json':
g.dashboard.new(this.config.dashboardNamePrefix + ' overview')
+ g.dashboard.withPanels(
g.util.panel.resolveCollapsedFlagOnRows(
g.util.grid.wrapPanels([
this.grafana.rows.requestsRow,
this.grafana.rows.networkRow,
this.grafana.rows.sessionsRow,
])
)
) + root.applyCommon(
vars.multiInstance + [
g.dashboard.variable.query.new('server')
+ g.dashboard.variable.custom.selectionOptions.withMulti(true)
+ g.dashboard.variable.query.queryTypes.withLabelValues(label='server', metric='wildfly_undertow_request_count_total{%(queriesSelectorGroupOnly)s}' % vars)
+ g.dashboard.variable.query.withDatasourceFromVariable(vars.datasources.prometheus),
g.dashboard.variable.query.new('deployment')
+ g.dashboard.variable.custom.selectionOptions.withMulti(true)
+ g.dashboard.variable.query.queryTypes.withLabelValues(label='deployment', metric='wildfly_undertow_active_sessions{%(queriesSelectorGroupOnly)s}' % vars)
+ g.dashboard.variable.query.withDatasourceFromVariable(vars.datasources.prometheus),
],
uid + '-overview',
tags,
links { wildflyOverview+:: {} },
annotations,
timezone,
refresh,
period,
),
'wildfly-datasource.json':
g.dashboard.new(this.config.dashboardNamePrefix + ' datasource')
+ g.dashboard.withPanels(
g.util.panel.resolveCollapsedFlagOnRows(
g.util.grid.wrapPanels([
this.grafana.rows.connectionsRow,
this.grafana.rows.transactionsRow,
])
)
) + root.applyCommon(
vars.multiInstance + [
g.dashboard.variable.query.new('datasource')
+ g.dashboard.variable.custom.selectionOptions.withMulti(true)
+ g.dashboard.variable.query.queryTypes.withLabelValues(label='data_source', metric='wildfly_datasources_pool_in_use_count{%(queriesSelectorGroupOnly)s}' % vars)
+ g.dashboard.variable.query.withDatasourceFromVariable(vars.datasources.prometheus),
],
uid + '-datasource',
tags,
links { wildflyDatasource+:: {} },
annotations,
timezone,
refresh,
period,
),

} + if this.config.enableLokiLogs then {
'wildfly-logs.json':
logslib.new(
this.config.dashboardNamePrefix + ' logs',
datasourceName=this.grafana.variables.datasources.loki.name,
datasourceRegex=this.grafana.variables.datasources.loki.regex,
filterSelector=this.config.filteringSelector,
labels=this.config.groupLabels + this.config.extraLogLabels,
formatParser=null,
showLogsVolume=this.config.showLogsVolume,
)
{
dashboards+:
{
logs+:
root.applyCommon(super.logs.templating.list, uid=uid + '-logs', tags=tags, links=links { logs+:: {} }, annotations=annotations, timezone=timezone, refresh=refresh, period=period),
},
panels+:
{
logs+:
g.panel.logs.options.withEnableLogDetails(true)
+ g.panel.logs.options.withShowTime(false)
+ g.panel.logs.options.withWrapLogMessage(false),
},
variables+: {
toArray+: [
this.grafana.variables.datasources.prometheus { hide: 2 },
],
},
}.dashboards.logs,

} else {},

applyCommon(vars, uid, tags, links, annotations, timezone, refresh, period):
g.dashboard.withTags(tags)
+ g.dashboard.withUid(uid)
+ g.dashboard.withLinks(std.objectValues(links))
+ g.dashboard.withTimezone(timezone)
+ g.dashboard.withRefresh(refresh)
+ g.dashboard.time.withFrom(period)
+ g.dashboard.withVariables(vars)
+ g.dashboard.withAnnotations(std.objectValues(annotations)),
}
2 changes: 0 additions & 2 deletions wildfly-mixin/dashboards/dashboards.libsonnet

This file was deleted.

Loading
Loading