Skip to content

Commit 91c3aeb

Browse files
BlargianBlargian
authored
Merge pull request #4383 from ClickHouse/move_otel_config
ClickStack: move otel config to snippet
2 parents 9225cea + 25ddf5a commit 91c3aeb

File tree

8 files changed

+169
-1848
lines changed

8 files changed

+169
-1848
lines changed

‎contribute/style-guide.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ Code will be inserted here
145145
\```
146146
```
147147

148+
**File paths are relative to the root of the docs repository**
149+
148150
You should commit the code inserted to the snippet as we want people (or LLMs)
149151
reading the markdown to be able to see the code. The advantage of importing code
150152
to snippets this way is that you can test your snippets externally or store them

‎docs/use-cases/observability/clickstack/deployment/_snippets/otel-cloud-config.yaml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
receivers:
2+
otlp/hyperdx:
3+
protocols:
4+
grpc:
5+
include_metadata: true
6+
endpoint: '0.0.0.0:4317'
7+
http:
8+
cors:
9+
allowed_origins: ['*']
10+
allowed_headers: ['*']
11+
include_metadata: true
12+
endpoint: '0.0.0.0:4318'
13+
processors:
14+
transform:
15+
log_statements:
16+
- context: log
17+
error_mode: ignore
18+
statements:
19+
# JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or
20+
# as a string containing JSON content.
21+
- set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where
22+
IsString(log.body)
23+
- merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert")
24+
where IsMap(log.cache)
25+
- flatten(log.attributes) where IsMap(log.cache)
26+
- merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body)
27+
- context: log
28+
error_mode: ignore
29+
conditions:
30+
- severity_number == 0 and severity_text == ""
31+
statements:
32+
# Infer: extract the first log level keyword from the first 256 characters of the body
33+
- set(log.cache["substr"], log.body.string) where Len(log.body.string)
34+
< 256
35+
- set(log.cache["substr"], Substring(log.body.string, 0, 256)) where
36+
Len(log.body.string) >= 256
37+
- set(log.cache, ExtractPatterns(log.cache["substr"],
38+
"(?i)(?P<0>(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace))"))
39+
# Infer: detect FATAL
40+
- set(log.severity_number, SEVERITY_NUMBER_FATAL) where
41+
IsMatch(log.cache["0"], "(?i)(alert|crit|emerg|fatal)")
42+
- set(log.severity_text, "fatal") where log.severity_number ==
43+
SEVERITY_NUMBER_FATAL
44+
# Infer: detect ERROR
45+
- set(log.severity_number, SEVERITY_NUMBER_ERROR) where
46+
IsMatch(log.cache["0"], "(?i)(error|err)")
47+
- set(log.severity_text, "error") where log.severity_number ==
48+
SEVERITY_NUMBER_ERROR
49+
# Infer: detect WARN
50+
- set(log.severity_number, SEVERITY_NUMBER_WARN) where
51+
IsMatch(log.cache["0"], "(?i)(warn|notice)")
52+
- set(log.severity_text, "warn") where log.severity_number ==
53+
SEVERITY_NUMBER_WARN
54+
# Infer: detect DEBUG
55+
- set(log.severity_number, SEVERITY_NUMBER_DEBUG) where
56+
IsMatch(log.cache["0"], "(?i)(debug|dbug)")
57+
- set(log.severity_text, "debug") where log.severity_number ==
58+
SEVERITY_NUMBER_DEBUG
59+
# Infer: detect TRACE
60+
- set(log.severity_number, SEVERITY_NUMBER_TRACE) where
61+
IsMatch(log.cache["0"], "(?i)(trace)")
62+
- set(log.severity_text, "trace") where log.severity_number ==
63+
SEVERITY_NUMBER_TRACE
64+
# Infer: else
65+
- set(log.severity_text, "info") where log.severity_number == 0
66+
- set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == 0
67+
- context: log
68+
error_mode: ignore
69+
statements:
70+
# Normalize the severity_text case
71+
- set(log.severity_text, ConvertCase(log.severity_text, "lower"))
72+
resourcedetection:
73+
detectors:
74+
- env
75+
- system
76+
- docker
77+
timeout: 5s
78+
override: false
79+
batch:
80+
memory_limiter:
81+
# 80% of maximum memory up to 2G, adjust for low memory environments
82+
limit_mib: 1500
83+
# 25% of limit up to 2G, adjust for low memory environments
84+
spike_limit_mib: 512
85+
check_interval: 5s
86+
connectors:
87+
routing/logs:
88+
default_pipelines: [logs/out-default]
89+
error_mode: ignore
90+
table:
91+
- context: log
92+
statement: route() where IsMatch(attributes["rr-web.event"], ".*")
93+
pipelines: [logs/out-rrweb]
94+
exporters:
95+
debug:
96+
verbosity: detailed
97+
sampling_initial: 5
98+
sampling_thereafter: 200
99+
clickhouse/rrweb:
100+
database: ${env:CLICKHOUSE_DATABASE}
101+
endpoint: ${env:CLICKHOUSE_ENDPOINT}
102+
password: ${env:CLICKHOUSE_PASSWORD}
103+
username: ${env:CLICKHOUSE_USER}
104+
ttl: 720h
105+
logs_table_name: hyperdx_sessions
106+
timeout: 5s
107+
retry_on_failure:
108+
enabled: true
109+
initial_interval: 5s
110+
max_interval: 30s
111+
max_elapsed_time: 300s
112+
clickhouse:
113+
database: ${env:CLICKHOUSE_DATABASE}
114+
endpoint: ${env:CLICKHOUSE_ENDPOINT}
115+
password: ${env:CLICKHOUSE_PASSWORD}
116+
username: ${env:CLICKHOUSE_USER}
117+
ttl: 720h
118+
timeout: 5s
119+
retry_on_failure:
120+
enabled: true
121+
initial_interval: 5s
122+
max_interval: 30s
123+
max_elapsed_time: 300s
124+
extensions:
125+
health_check:
126+
endpoint: :13133
127+
service:
128+
pipelines:
129+
traces:
130+
receivers: [otlp/hyperdx]
131+
processors: [memory_limiter, batch]
132+
exporters: [clickhouse]
133+
metrics:
134+
receivers: [otlp/hyperdx]
135+
processors: [memory_limiter, batch]
136+
exporters: [clickhouse]
137+
logs/in:
138+
receivers: [otlp/hyperdx]
139+
exporters: [routing/logs]
140+
logs/out-default:
141+
receivers: [routing/logs]
142+
processors: [memory_limiter, transform, batch]
143+
exporters: [clickhouse]
144+
logs/out-rrweb:
145+
receivers: [routing/logs]
146+
processors: [memory_limiter, batch]
147+
exporters: [clickhouse/rrweb]

‎docs/use-cases/observability/clickstack/deployment/hyperdx-clickhouse-cloud.md‎

Lines changed: 12 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -59,157 +59,17 @@ This step ensures tables are created with an Open Telemetry (OTel) schema, which
5959
The following instructions use the standard distribution of the OTel collector, rather than the ClickStack distribution. The latter requires an OpAMP server for configuration. This is currently not supported in private preview. The configuration below replicates the version used by the ClickStack distribution of the collector, providing an OTLP endpoint to which events can be sent.
6060
:::
6161

62-
Create a `otel-file-collector.yaml` file with the following content:
63-
64-
```yaml
65-
receivers:
66-
otlp/hyperdx:
67-
protocols:
68-
grpc:
69-
include_metadata: true
70-
endpoint: '0.0.0.0:4317'
71-
http:
72-
cors:
73-
allowed_origins: ['*']
74-
allowed_headers: ['*']
75-
include_metadata: true
76-
endpoint: '0.0.0.0:4318'
77-
processors:
78-
transform:
79-
log_statements:
80-
- context: log
81-
error_mode: ignore
82-
statements:
83-
# JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or
84-
# as a string containing JSON content.
85-
- set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where
86-
IsString(log.body)
87-
- merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert")
88-
where IsMap(log.cache)
89-
- flatten(log.attributes) where IsMap(log.cache)
90-
- merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body)
91-
- context: log
92-
error_mode: ignore
93-
conditions:
94-
- severity_number == 0 and severity_text == ""
95-
statements:
96-
# Infer: extract the first log level keyword from the first 256 characters of the body
97-
- set(log.cache["substr"], log.body.string) where Len(log.body.string)
98-
< 256
99-
- set(log.cache["substr"], Substring(log.body.string, 0, 256)) where
100-
Len(log.body.string) >= 256
101-
- set(log.cache, ExtractPatterns(log.cache["substr"],
102-
"(?i)(?P<0>(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace))"))
103-
# Infer: detect FATAL
104-
- set(log.severity_number, SEVERITY_NUMBER_FATAL) where
105-
IsMatch(log.cache["0"], "(?i)(alert|crit|emerg|fatal)")
106-
- set(log.severity_text, "fatal") where log.severity_number ==
107-
SEVERITY_NUMBER_FATAL
108-
# Infer: detect ERROR
109-
- set(log.severity_number, SEVERITY_NUMBER_ERROR) where
110-
IsMatch(log.cache["0"], "(?i)(error|err)")
111-
- set(log.severity_text, "error") where log.severity_number ==
112-
SEVERITY_NUMBER_ERROR
113-
# Infer: detect WARN
114-
- set(log.severity_number, SEVERITY_NUMBER_WARN) where
115-
IsMatch(log.cache["0"], "(?i)(warn|notice)")
116-
- set(log.severity_text, "warn") where log.severity_number ==
117-
SEVERITY_NUMBER_WARN
118-
# Infer: detect DEBUG
119-
- set(log.severity_number, SEVERITY_NUMBER_DEBUG) where
120-
IsMatch(log.cache["0"], "(?i)(debug|dbug)")
121-
- set(log.severity_text, "debug") where log.severity_number ==
122-
SEVERITY_NUMBER_DEBUG
123-
# Infer: detect TRACE
124-
- set(log.severity_number, SEVERITY_NUMBER_TRACE) where
125-
IsMatch(log.cache["0"], "(?i)(trace)")
126-
- set(log.severity_text, "trace") where log.severity_number ==
127-
SEVERITY_NUMBER_TRACE
128-
# Infer: else
129-
- set(log.severity_text, "info") where log.severity_number == 0
130-
- set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == 0
131-
- context: log
132-
error_mode: ignore
133-
statements:
134-
# Normalize the severity_text case
135-
- set(log.severity_text, ConvertCase(log.severity_text, "lower"))
136-
resourcedetection:
137-
detectors:
138-
- env
139-
- system
140-
- docker
141-
timeout: 5s
142-
override: false
143-
batch:
144-
memory_limiter:
145-
# 80% of maximum memory up to 2G, adjust for low memory environments
146-
limit_mib: 1500
147-
# 25% of limit up to 2G, adjust for low memory environments
148-
spike_limit_mib: 512
149-
check_interval: 5s
150-
connectors:
151-
routing/logs:
152-
default_pipelines: [logs/out-default]
153-
error_mode: ignore
154-
table:
155-
- context: log
156-
statement: route() where IsMatch(attributes["rr-web.event"], ".*")
157-
pipelines: [logs/out-rrweb]
158-
exporters:
159-
debug:
160-
verbosity: detailed
161-
sampling_initial: 5
162-
sampling_thereafter: 200
163-
clickhouse/rrweb:
164-
database: ${env:CLICKHOUSE_DATABASE}
165-
endpoint: ${env:CLICKHOUSE_ENDPOINT}
166-
password: ${env:CLICKHOUSE_PASSWORD}
167-
username: ${env:CLICKHOUSE_USER}
168-
ttl: 720h
169-
logs_table_name: hyperdx_sessions
170-
timeout: 5s
171-
retry_on_failure:
172-
enabled: true
173-
initial_interval: 5s
174-
max_interval: 30s
175-
max_elapsed_time: 300s
176-
clickhouse:
177-
database: ${env:CLICKHOUSE_DATABASE}
178-
endpoint: ${env:CLICKHOUSE_ENDPOINT}
179-
password: ${env:CLICKHOUSE_PASSWORD}
180-
username: ${env:CLICKHOUSE_USER}
181-
ttl: 720h
182-
timeout: 5s
183-
retry_on_failure:
184-
enabled: true
185-
initial_interval: 5s
186-
max_interval: 30s
187-
max_elapsed_time: 300s
188-
extensions:
189-
health_check:
190-
endpoint: :13133
191-
service:
192-
pipelines:
193-
traces:
194-
receivers: [otlp/hyperdx]
195-
processors: [memory_limiter, batch]
196-
exporters: [clickhouse]
197-
metrics:
198-
receivers: [otlp/hyperdx]
199-
processors: [memory_limiter, batch]
200-
exporters: [clickhouse]
201-
logs/in:
202-
receivers: [otlp/hyperdx]
203-
exporters: [routing/logs]
204-
logs/out-default:
205-
receivers: [routing/logs]
206-
processors: [memory_limiter, transform, batch]
207-
exporters: [clickhouse]
208-
logs/out-rrweb:
209-
receivers: [routing/logs]
210-
processors: [memory_limiter, batch]
211-
exporters: [clickhouse/rrweb]
62+
Download the configuration for the OTel collector:
63+
64+
```bash
65+
curl -O https://raw.githubusercontent.com/ClickHouse/clickhouse-docs/refs/heads/main/docs/use-cases/observability/clickstack/deployment/_snippets/otel-cloud-collector.yaml
66+
```
67+
68+
<details>
69+
<summary>otel-cloud-collector.yaml</summary>
70+
```yaml file=docs/use-cases/observability/clickstack/deployment/_snippets/otel-cloud-config.yaml
21271
```
72+
</details>
21373

21474
Deploy the collector using the following Docker command, setting the respective environment variables to the connection settings recorded earlier and using the appropriate command below based on your operating system.
21575

@@ -228,7 +88,7 @@ docker run --rm -it \
22888
-e CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD} \
22989
-e CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE} \
23090
--user 0:0 \
231-
-v "$(pwd)/otel-file-collector.yaml":/etc/otel/config.yaml \
91+
-v "$(pwd)/otel-cloud-collector.yaml":/etc/otel/config.yaml \
23292
-v /var/log:/var/log:ro \
23393
-v /private/var/log:/private/var/log:ro \
23494
otel/opentelemetry-collector-contrib:latest \
@@ -242,7 +102,7 @@ docker run --rm -it \
242102
# -e CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD} \
243103
# -e CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE} \
244104
# --user 0:0 \
245-
# -v "$(pwd)/otel-file-collector.yaml":/etc/otel/config.yaml \
105+
# -v "$(pwd)/otel-cloud-collector.yaml":/etc/otel/config.yaml \
246106
# -v /var/log:/var/log:ro \
247107
# -v /private/var/log:/private/var/log:ro \
248108
# otel/opentelemetry-collector-contrib:latest \

‎docs/use-cases/observability/clickstack/deployment/local-mode-only.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ Similar to the [all-in-one image](/use-cases/observability/clickstack/deployment
3434

3535
### Deploy with Docker {#deploy-with-docker}
3636

37-
Local mode deploys the HyperDX UI only, accessible on port 8080.
37+
Local mode deploys the HyperDX UI on port 8080.
3838

3939
```shell
4040
docker run -p 8080:8080 docker.hyperdx.io/hyperdx/hyperdx-local

0 commit comments

Comments
 (0)