Skip to content
This repository was archived by the owner on Dec 14, 2022. It is now read-only.

Commit 13bebca

Browse files
author
Chris Wiechmann
committed
Refactored
1 parent 3966e3f commit 13bebca

File tree

2 files changed

+38
-38
lines changed

2 files changed

+38
-38
lines changed

.github/workflows/logstash.yml

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,23 +40,23 @@ jobs:
4040
run: |
4141
wget -c https://github.com/magnusbaeck/logstash-filter-verifier/releases/download/1.6.3/logstash-filter-verifier_1.6.3_linux_amd64.tar.gz -O - | tar -xvz
4242
- name: Test Logstash pipeline
43-
# run: |
44-
# echo Using API_BUILDER_URL: $API_BUILDER_URL
45-
# echo Using MEMCACHED: $MEMCACHED
46-
# echo Using GEOIP_ENABLED: $GEOIP_ENABLED, GEOIP_CUSTOM_ATTRIBUTE: ${GEOIP_CUSTOM_ATTRIBUTE}, GEOIP_CACHE_SIZE: ${GEOIP_CACHE_SIZE}
47-
# ./logstash-filter-verifier --diff-command="diff -y" --logstash-output --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=CACHE_API_PATHS ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
48-
4943
run: |
5044
echo Using API_BUILDER_URL: $API_BUILDER_URL
5145
echo Using MEMCACHED: $MEMCACHED
52-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/jms/test-opentrafficlog-jms.json ./logstash/pipelines/OpenTrafficPipeline.conf
53-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/beatsInput/test-index-creation-lookup.json ./logstash/pipelines/BeatsInputPipeline.conf
54-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=DROP_TRACE_MESSAGE_LEVELS ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
55-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages-gmt-4.json ./logstash/pipelines/TraceMessagesPipeline.conf
56-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE --keep-env=EVENTLOG_CUSTOM_ATTR ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
57-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=CACHE_API_PATHS ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
58-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/filetransfer/test-opentrafficlog-filetransfer.json ./logstash/pipelines/OpenTrafficPipeline.conf
59-
./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/domainAudit/test-domain-audit-events.json ./logstash/pipelines/DomainAuditPipeline.conf
46+
echo Using GEOIP_ENABLED: $GEOIP_ENABLED, GEOIP_CUSTOM_ATTRIBUTE: ${GEOIP_CUSTOM_ATTRIBUTE}, GEOIP_CACHE_SIZE: ${GEOIP_CACHE_SIZE}
47+
./logstash-filter-verifier --diff-command="diff -y" --logstash-output --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=CACHE_API_PATHS ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
48+
49+
# run: |
50+
# echo Using API_BUILDER_URL: $API_BUILDER_URL
51+
# echo Using MEMCACHED: $MEMCACHED
52+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/jms/test-opentrafficlog-jms.json ./logstash/pipelines/OpenTrafficPipeline.conf
53+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/beatsInput/test-index-creation-lookup.json ./logstash/pipelines/BeatsInputPipeline.conf
54+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=DROP_TRACE_MESSAGE_LEVELS ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
55+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages-gmt-4.json ./logstash/pipelines/TraceMessagesPipeline.conf
56+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE --keep-env=EVENTLOG_CUSTOM_ATTR ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
57+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=CACHE_API_PATHS ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
58+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/filetransfer/test-opentrafficlog-filetransfer.json ./logstash/pipelines/OpenTrafficPipeline.conf
59+
# ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/domainAudit/test-domain-audit-events.json ./logstash/pipelines/DomainAuditPipeline.conf
6060
env:
6161
API_BUILDER_URL: 'http://localhost:8080'
6262
MEMCACHED: 'localhost:11211'

logstash/pipelines/OpenTrafficPipeline.conf

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,30 @@ filter {
2424
add_field => { "[@metadata][ignoreField][apiPath]" => "" } # Default values for apiPath
2525
add_field => { "[@metadata][ignoreField][policyName]" => "" } # and policyName used for ingoreLookup
2626
}
27+
# Create a cache key for the API either on the complete received request path or if configured, only using a specific part of the path.
28+
# This prevents API requests with path parameters from not being cached efficiently due to their variable parameter.
29+
# Example: API-Request: /v1/get/pet/687687678, Configured-Path: /v1/get/pet - API-Details cached based on Configured-Path (/v1/get/pet)
30+
ruby {
31+
id => "Set API-Cache-Key prefix"
32+
code => '
33+
cacheAPIPaths = event.get("[@metadata][cacheAPIPaths]");
34+
apiRequestPath = event.get("[transactionSummary][path]");
35+
if (cacheAPIPaths.nil? || cacheAPIPaths.empty?)
36+
event.set("[@metadata][apiCacheKeyPrefix]", apiRequestPath);
37+
return;
38+
end
39+
apiName = event.get("[transactionSummary][serviceContext][service]");
40+
logger.debug("Configured paths and current API-Request path: ", { "configuredPaths" => cacheAPIPaths, "apiRequestPath" => apiRequestPath } );
41+
event.set("[@metadata][apiCacheKeyPrefix]", apiRequestPath);
42+
for configuredPath in cacheAPIPaths.split(",") do
43+
if(apiRequestPath.start_with?(configuredPath) )
44+
logger.debug("Using configured path as primary cache key because it matches to request path. ", { "configuredPath" => configuredPath, "apiRequestPath" => apiRequestPath });
45+
event.set("[@metadata][apiCacheKeyPrefix]", configuredPath);
46+
break;
47+
end
48+
end
49+
'
50+
}
2751
# First, is to check if the document should be ignored/dropped (configured based on the apiPath or PolicyName)
2852
if([transactionSummary] and [transactionSummary][path]) { # Only if a path is given (not given for instance for Scheduled-Policies)
2953
mutate { replace => { "[@metadata][ignoreField][apiPath]" => "%{[transactionSummary][path]}" } }
@@ -147,30 +171,6 @@ filter {
147171
# Without an API name, only API path is used for the API lookup.
148172
mutate { add_field => { "[@metadata][apiName]" => "" } }
149173
}
150-
# Create a cache key for the API either on the complete received request path or if configured, only using a specific part of the path.
151-
# This prevents API requests with path parameters from not being cached efficiently due to their variable parameter.
152-
# Example: API-Request: /v1/get/pet/687687678, Configured-Path: /v1/get/pet - API-Details cached based on Configured-Path (/v1/get/pet)
153-
ruby {
154-
id => "Set API-Cache-Key prefix"
155-
code => '
156-
cacheAPIPaths = event.get("[@metadata][cacheAPIPaths]");
157-
apiRequestPath = event.get("[transactionSummary][path]");
158-
if (cacheAPIPaths.nil? || cacheAPIPaths.empty?)
159-
event.set("[@metadata][apiCacheKeyPrefix]", apiRequestPath);
160-
return;
161-
end
162-
apiName = event.get("[transactionSummary][serviceContext][service]");
163-
logger.debug("Configured paths and current API-Request path: ", { "configuredPaths" => cacheAPIPaths, "apiRequestPath" => apiRequestPath } );
164-
event.set("[@metadata][apiCacheKeyPrefix]", apiRequestPath);
165-
for configuredPath in cacheAPIPaths.split(",") do
166-
if(apiRequestPath.start_with?(configuredPath) )
167-
logger.debug("Using configured path as primary cache key because it matches to request path. ", { "configuredPath" => configuredPath, "apiRequestPath" => apiRequestPath });
168-
event.set("[@metadata][apiCacheKeyPrefix]", configuredPath);
169-
break;
170-
end
171-
end
172-
'
173-
}
174174
mutate {
175175
id => "Set API-Cache-Key"
176176
add_field => {

0 commit comments

Comments
 (0)