Skip to content

Commit 707db15

Browse files
jcpunkjcpunk
committed
Initial commit of cvmfs client prometheus exporter
Signed-off-by: Pat Riehecky <riehecky@fnal.gov>
1 parent 93518d9 commit 707db15

File tree

3 files changed

+350
-0
lines changed

3 files changed

+350
-0
lines changed

‎cvmfs-client-prometheus.sh‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
#!/bin/bash -u
2+
3+
HTTP_HEADER='FALSE'
4+
5+
TMPFILE=$(mktemp)
6+
7+
cleanup_tmpfile() {
8+
if [ -n "${TMPFILE}" ] && [ -f "${TMPFILE}" ]; then
9+
rm -f "${TMPFILE}"
10+
fi
11+
}
12+
trap cleanup_tmpfile EXIT
13+
14+
# CVMFS Extended Attributes and their descriptions
15+
declare -A CVMFS_EXTENDED_ATTRIBUTE_GAGUES=(
16+
['hitrate']='CVMFS cache hit rate (%)'
17+
['inode_max']='Shows the highest possible inode with the current set of loaded catalogs.'
18+
['maxfd']='Shows the maximum number of file descriptors available to file system clients.'
19+
['ncleanup24']='Shows the number of cache cleanups in the last 24 hours.'
20+
['nclg']='Shows the number of currently loaded nested catalogs.'
21+
['ndiropen']='Shows the overall number of opened directories.'
22+
['pid']='Shows the process id of the CernVM-FS Fuse process.'
23+
['speed']='Shows the average download speed.'
24+
['useddirp']='Shows the number of file descriptors currently issued to file system clients.'
25+
['usedfd']='Shows the number of open directories currently used by file system clients.'
26+
)
27+
28+
#############################################################
29+
usage() {
30+
echo "Usage: $0 [-h|--help] [--http]" >&2
31+
echo '' >&2
32+
echo ' --http: add the HTTP protocol header to the output' >&2
33+
echo '' >&2
34+
echo 'NOTE: The user running this script must have read access' >&2
35+
echo ' to the CVMFS cache files!' >&2
36+
exit 1
37+
}
38+
39+
generate_metric() {
40+
local metric_name="$1"
41+
local metric_type="$2"
42+
local help_text="$3"
43+
local metric_labels="$4"
44+
local metric_value="$5"
45+
46+
cat >>"${TMPFILE}" <<EOF
47+
# HELP $metric_name $help_text
48+
# TYPE $metric_name $metric_type
49+
${metric_name}{${metric_labels}} ${metric_value}
50+
EOF
51+
}
52+
53+
list_mounted_cvmfs_repos() {
54+
cvmfs_config status | tr -s '[:space:]' | cut -d ' ' -f 1 | sort -u
55+
}
56+
57+
mountpoint_for_cvmfs_repo() {
58+
local reponame
59+
reponame="$1"
60+
61+
cvmfs_talk -i "${reponame}" mountpoint
62+
}
63+
64+
fqrn_for_cvmfs_repo() {
65+
local reponame
66+
reponame="$1"
67+
68+
local repopath
69+
repopath=$(mountpoint_for_cvmfs_repo "${reponame}")
70+
71+
attr -g fqrn "${repopath}" | tail -n +2
72+
}
73+
74+
get_cvmfs_repo_extended_attribute_gague_metrics() {
75+
local reponame
76+
reponame="$1"
77+
78+
local repomountpoint
79+
repomountpoint=$(mountpoint_for_cvmfs_repo "${reponame}")
80+
81+
local fqrn
82+
fqrn=$(fqrn_for_cvmfs_repo "${reponame}")
83+
84+
local attribute
85+
for attribute in "${!CVMFS_EXTENDED_ATTRIBUTE_GAGUES[@]}"; do
86+
local result
87+
result=$(attr -g "${attribute}" "${repomountpoint}" | tail -n +2)
88+
generate_metric "cvmfs_${attribute}" 'gague' "${CVMFS_EXTENDED_ATTRIBUTE_GAGUES[${attribute}]}" "repo=${fqrn}" "${result}"
89+
done
90+
}
91+
92+
get_cvmfs_repo_proxy_metrics() {
93+
local reponame
94+
reponame="$1"
95+
96+
local repomountpoint
97+
repomountpoint=$(mountpoint_for_cvmfs_repo "${reponame}")
98+
99+
local fqrn
100+
fqrn=$(fqrn_for_cvmfs_repo "${reponame}")
101+
102+
local proxy_list
103+
mapfile -t proxy_list < <(attr -g proxy_list "${repomountpoint}" | tail -n +2 | grep -v '^$')
104+
105+
local proxy_filter_by_group
106+
mapfile -t proxy_filter_by_group < <(cvmfs_talk -i "${reponame}" proxy info | tail -n +2 | grep '^\[' | grep ']' | tr -s '[:space:]')
107+
108+
local proxy
109+
local my_proxy_group
110+
for proxy in "${proxy_list[@]}"; do
111+
local line
112+
local result
113+
for line in "${proxy_filter_by_group[@]}"; do
114+
result=$(echo "${line}" | grep "${proxy}" | cut -d' ' -f 1 | tr -d '][')
115+
if [[ "x${result}" != 'x' ]]; then
116+
my_proxy_group=${result}
117+
break
118+
fi
119+
done
120+
generate_metric "cvmfs_proxy" "gague" "Shows all registered proxies for this repository." "repo=${fqrn},group=${my_proxy_group},url=${proxy}" 1
121+
done
122+
}
123+
124+
get_cvmfs_repo_metrics() {
125+
local reponame
126+
reponame="$1"
127+
128+
local repomountpoint
129+
repomountpoint=$(mountpoint_for_cvmfs_repo "${reponame}")
130+
131+
local fqrn
132+
fqrn=$(fqrn_for_cvmfs_repo "${reponame}")
133+
134+
local repo_pid
135+
repo_pid=$(cvmfs_talk -i "${reponame}" pid)
136+
137+
local cache_volume
138+
cache_volume=$(cvmfs_talk -i "${reponame}" parameters | grep CVMFS_CACHE_BASE | tr '=' ' ' | tr -s '[:space:]' | cut -d ' ' -f 2)
139+
140+
local cached_bytes
141+
cached_bytes=$(cvmfs_talk -i "${reponame}" cache size | tr -d ')(' | tr -s '[:space:]' | cut -d ' ' -f 6)
142+
generate_metric 'cvmfs_cached_bytes' 'gauge' 'CVMFS currently cached bytes.' "repo=${fqrn}" "${cached_bytes}"
143+
144+
local pinned_bytes
145+
pinned_bytes=$(cvmfs_talk -i "${reponame}" cache size | tr -d ')(' | tr -s '[:space:]' | cut -d ' ' -f 10)
146+
generate_metric 'cvmfs_pinned_bytes' 'gauge' 'CVMFS currently pinned bytes.' "repo=${fqrn}" "${pinned_bytes}"
147+
148+
local total_cache_size_mb
149+
total_cache_size_mb=$(cvmfs_talk -i "${reponame}" parameters | grep CVMFS_QUOTA_LIMIT | tr '=' ' ' | tr -s '[:space:]' | cut -d ' ' -f 2)
150+
local total_cache_size
151+
total_cache_size=$((total_cache_size_mb * 1024 * 1024))
152+
generate_metric 'cvmfs_total_cache_size_bytes' 'gague' 'CVMFS configured cache size via CVMFS_QUOTA_LIMIT.' "repo=${fqrn}" "${total_cache_size}"
153+
154+
local cache_volume_max
155+
cache_volume_max=$(df -B1 "${cache_volume}" | tail -n 1 | tr -s '[:space:]' | cut -d ' ' -f 2)
156+
generate_metric 'cvmfs_physical_cache_size_bytes' 'gague' 'CVMFS cache volume physical size.' "repo=${fqrn}" "${cache_volume_max}"
157+
158+
local cache_volume_free
159+
cache_volume_free=$(df -B1 "${cache_volume}" | tail -n 1 | tr -s '[:space:]' | cut -d ' ' -f 4)
160+
generate_metric 'cvmfs_physical_cache_avail_bytes' 'gague' 'CVMFS cache volume physical free space available.' "repo=${fqrn}" "${cache_volume_free}"
161+
162+
local cvmfs_mount_version
163+
cvmfs_mount_version=$(attr -g version "${repomountpoint}" | tail -n +2)
164+
local cvmfs_mount_revision
165+
cvmfs_mount_revision=$(attr -g revision "${repomountpoint}" | tail -n +2)
166+
generate_metric 'cvmfs_repo' 'guage' 'Shows the version of CVMFS used by this repository.' "repo=${fqrn},mountpoint=${repomountpoint},version=${cvmfs_mount_version},revision=${cvmfs_mount_revision}" 1
167+
168+
local cvmfs_mount_rx_kb
169+
cvmfs_mount_rx_kb=$(attr -g rx "${repomountpoint}" | tail -n +2)
170+
local cvmfs_mount_rx
171+
cvmfs_mount_rx=$((cvmfs_mount_rx_kb * 1024))
172+
generate_metric 'cvmfs_rx_total' 'counter' 'Shows the overall amount of downloaded bytes since mounting.' "repo=${fqrn}" "${cvmfs_mount_rx}"
173+
174+
local cvmfs_mount_uptime_minutes
175+
cvmfs_mount_uptime_minutes=$(attr -g uptime "${repomountpoint}" | tail -n +2)
176+
local now
177+
local rounded_now_to_minute
178+
local cvmfs_mount_uptime
179+
local cvmfs_mount_epoch_time
180+
now=$(date +%s)
181+
rounded_now_to_minute=$((now - (now % 60)))
182+
cvmfs_mount_uptime=$((cvmfs_mount_uptime_minutes * 60))
183+
cvmfs_mount_epoch_time=$((rounded_now_to_minute - cvmfs_mount_uptime))
184+
generate_metric 'cvmfs_uptime_seconds' 'counter' 'Shows the time since the repo was mounted.' "repo=${fqrn}" "${cvmfs_mount_uptime}"
185+
generate_metric 'cvmfs_mount_epoch_timestamp' 'counter' 'Shows the epoch time the repo was mounted.' "repo=${fqrn}" "${cvmfs_mount_epoch_time}"
186+
187+
local cvmfs_repo_expires_min
188+
cvmfs_repo_expires_min=$(attr -g expires "${repomountpoint}" | tail -n +2)
189+
local cvmfs_repo_expires
190+
cvmfs_repo_expires=$((cvmfs_repo_expires_min * 60))
191+
generate_metric 'cvmfs_repo_expires_seconds' 'gague' 'Shows the remaining life time of the mounted root file catalog in seconds.' "repo=${fqrn}" "${cvmfs_repo_expires}"
192+
193+
local cvmfs_mount_ndownload
194+
cvmfs_mount_ndownload=$(attr -g ndownload "${repomountpoint}" | tail -n +2)
195+
generate_metric 'cvmfs_ndownload_total' 'counter' 'Shows the overall number of downloaded files since mounting.' "repo=${fqrn}" "${cvmfs_mount_ndownload}"
196+
197+
local cvmfs_mount_nioerr
198+
cvmfs_mount_nioerr=$(attr -g nioerr "${repomountpoint}" | tail -n +2)
199+
generate_metric 'cvmfs_nioerr_total' 'counter' 'Shows the total number of I/O errors encountered since mounting.' "repo=${fqrn}" "${cvmfs_mount_nioerr}"
200+
201+
local cvmfs_mount_timeout
202+
cvmfs_mount_timeout=$(attr -g timeout "${repomountpoint}" | tail -n +2)
203+
generate_metric 'cvmfs_timeout' 'guage' 'Shows the timeout for proxied connections in seconds.' "repo=${fqrn}" "${cvmfs_mount_timeout}"
204+
205+
local cvmfs_mount_timeout_direct
206+
cvmfs_mount_timeout_direct=$(attr -g timeout_direct "${repomountpoint}" | tail -n +2)
207+
generate_metric 'cvmfs_timeout_direct' 'guage' 'Shows the timeout for direct connections in seconds.' "repo=${fqrn}" "${cvmfs_mount_timeout_direct}"
208+
209+
local cvmfs_mount_timestamp_last_ioerr
210+
cvmfs_mount_timestamp_last_ioerr=$(attr -g timestamp_last_ioerr "${repomountpoint}" | tail -n +2)
211+
generate_metric 'cvmfs_timestamp_last_ioerr' 'counter' 'Shows the timestamp of the last ioerror.' "repo=${fqrn}" "${cvmfs_mount_timestamp_last_ioerr}"
212+
213+
local cvmfs_repo_pid_statline
214+
cvmfs_repo_pid_statline=$(</proc/"${repo_pid}"/stat)
215+
local cvmfs_repo_stats
216+
read -ra cvmfs_repo_stats <<<"${cvmfs_repo_pid_statline}"
217+
local cvmfs_utime
218+
local cvmfs_stime
219+
cvmfs_utime=${cvmfs_repo_stats[13]}
220+
cvmfs_stime=${cvmfs_repo_stats[14]}
221+
local cvmfs_user_seconds
222+
local cvmfs_system_seconds
223+
cvmfs_user_seconds=$(printf "%.2f" "$(echo "scale=4; $cvmfs_utime / $CLOCK_TICK" | bc)")
224+
cvmfs_system_seconds=$(printf "%.2f" "$(echo "scale=4; $cvmfs_stime / $CLOCK_TICK" | bc)")
225+
generate_metric 'cvmfs_cpu_user_total' 'counter' 'CPU time used in userspace by CVMFS mount in seconds.' "repo=${fqrn}" "${cvmfs_user_seconds}"
226+
generate_metric 'cvmfs_cpu_system_total' 'counter' 'CPU time used in the kernel system calls by CVMFS mount in seconds.' "repo=${fqrn}" "${cvmfs_system_seconds}"
227+
228+
local cvmfs_mount_active_proxy
229+
cvmfs_mount_active_proxy=$(attr -g proxy "${repomountpoint}" | tail -n +2)
230+
generate_metric 'cvmfs_active_proxy' 'gauge' 'Shows the active proxy in use for this mount.' "repo=${fqrn},proxy=${cvmfs_mount_active_proxy}" 1
231+
232+
# Pull in xattr based metrics with simple labels
233+
get_cvmfs_repo_extended_attribute_gague_metrics "${reponame}"
234+
get_cvmfs_repo_proxy_metrics "${reponame}"
235+
}
236+
237+
#############################################################
238+
# List "uncommon" commands we expect
239+
for cmd in attr bc cvmfs_config cvmfs_talk grep; do
240+
if ! command -v "$cmd" &>/dev/null; then
241+
echo "ERROR: Required command '$cmd' not found" >&2
242+
exit 1
243+
fi
244+
done
245+
246+
#############################################################
247+
# setup args in the right order for making getopt evaluation
248+
# nice and easy. You'll need to read the manpages for more info
249+
args=$(getopt --options 'h' --longoptions 'help,http' -- "$@")
250+
eval set -- "$args"
251+
252+
for arg in $@; do
253+
case $1 in
254+
--)
255+
# end of getopt args, shift off the -- and get out of the loop
256+
shift
257+
break 2
258+
;;
259+
--http)
260+
# Add the http header to the output
261+
HTTP_HEADER='TRUE'
262+
shift
263+
;;
264+
-h | --help)
265+
# get help
266+
shift
267+
usage
268+
;;
269+
esac
270+
done
271+
272+
CLOCK_TICK=$(getconf CLK_TCK)
273+
274+
for REPO in $(cvmfs_config status | cut -d ' ' -f 1); do
275+
get_cvmfs_repo_metrics "${REPO}"
276+
done
277+
278+
if [[ "${HTTP_HEADER}" == 'TRUE' ]]; then
279+
content_length=$(stat --printf="%s" "${TMPFILE}")
280+
echo -ne "HTTP/1.1 200 OK\r\n"
281+
echo -ne "Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=underscores\r\n"
282+
echo -ne "Content-Length: ${content_length}\r\n"
283+
echo -ne "Connection: close\r\n"
284+
echo -ne "\r\n"
285+
fi
286+
287+
cat "${TMPFILE}"

‎systemd/cvmfs-client-prometheus.socket‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=Prometheus Exporter for CVMFS clients network socket
3+
4+
[Socket]
5+
ListenStream=9868
6+
Accept=true
7+
MaxConnections=30
8+
MaxConnectionsPerSource=10
9+
DeferAcceptSec=1
10+
11+
[Install]
12+
WantedBy=sockets.target

‎systemd/cvmfs-client-prometheus@.service‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
[Unit]
2+
Description=Prometheus Exporter for CVMFS clients
3+
After=network.target
4+
5+
[Service]
6+
ExecStart=-/usr/libexec/cvmfs/cvmfs-prometheus.sh --http
7+
StandardInput=null
8+
StandardOutput=socket
9+
StandardError=journal
10+
11+
User=cvmfs
12+
Group=cvmfs
13+
14+
WorkingDirectory=/
15+
CapabilityBoundingSet=
16+
AmbientCapabilities=
17+
18+
LimitNOFILE=1024
19+
TasksMax=1024
20+
Nice=5
21+
CPUWeight=30
22+
MemoryMax=32M
23+
IOSchedulingClass=best-effort
24+
IOSchedulingPriority=7
25+
IOWeight=30
26+
27+
SystemCallArchitectures=native
28+
SystemCallFilter=@system-service
29+
30+
LockPersonality=true
31+
MemoryDenyWriteExecute=true
32+
NoNewPrivileges=true
33+
PrivateDevices=true
34+
PrivateTmp=true
35+
ProtectClock=true
36+
ProtectControlGroups=true
37+
ProtectHome=true
38+
ProtectKernelLogs=true
39+
ProtectKernelModules=true
40+
ProtectKernelTunables=true
41+
ProtectSystem=strict
42+
RemoveIPC=true
43+
RestrictNamespaces=true
44+
RestrictRealtime=true
45+
RestrictSUIDSGID=true
46+
47+
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
48+
MountAPIVFS=false
49+
50+
[Install]
51+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)