New parameter: DROP_TRACE_MESSAGE_LEVELS added

Author: Chris Wiechmann

.github/workflows/logstash.yml

@@ -48,7 +48,7 @@ jobs:
         echo Using MEMCACHED: $MEMCACHED
         ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/jms/test-opentrafficlog-jms.json ./logstash/pipelines/OpenTrafficPipeline.conf
         ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/beatsInput/test-index-creation-lookup.json ./logstash/pipelines/BeatsInputPipeline.conf
-        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=DROP_TRACE_MESSAGE_LEVELS ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
         ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages-gmt-4.json ./logstash/pipelines/TraceMessagesPipeline.conf
         ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
         ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
@@ -57,6 +57,7 @@ jobs:
       env:
         API_BUILDER_URL: 'http://localhost:8080'
         MEMCACHED: 'localhost:11211'
+        DROP_TRACE_MESSAGE_LEVELS: DEBUG
         # This is not relevant as we test with HTTP
         # But it's required by Logstash to be a valid certificate
         API_BUILDER_SSL_CERT: "./config/certificates/apibuilder4elastic.crt"

CHANGELOG.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Added Quartely- and Yearly-API-Request dashboars in addition to the existing Real-Time Dashboard
 - Added support for Transactions [#83](https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/issues/83)
 - If Current-User request to ANM fails, API-Builder is now trying it again [#135](https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/issues/135)
+- New optional parameter: DROP_TRACE_MESSAGE_LEVELS to skip indexing of certain trace messages [#XXX](https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/issues/XXX)
 
 ## [3.3.2] 2021-08-30
 ### Fixed

UPDATE.md

@@ -55,7 +55,7 @@ On the other hand, the API builder Docker image, as a central component of the s
 | 3.3.0 | [X](#api-builderlogstashmemcached) | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | -               | [X](#parameters)|-                   | 7.12.1  |            |
 | 3.3.1 | [X](#api-builderlogstashmemcached) | -                                  | -                                  | -             | -               | -               | [X](#parameters)|-                   | 7.12.1  |            |
 | 3.3.2 | [X](#api-builderlogstashmemcached) | -                                  | -                                  | -             | -               | -               | -               |-                   | 7.12.1  |            |
-| 3.4.0 | [X](#api-builderlogstashmemcached) | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | [X](#dashboards)| -               |[X](#elastic-config)| 7.14.0  | Unreleased |
+| 3.4.0 | [X](#api-builderlogstashmemcached) | [X](#api-builderlogstashmemcached) | -                                  | -             | -               | [X](#dashboards)| [X](#parameters)|[X](#elastic-config)| 7.14.0  | Unreleased |
 
 ### Update from Version 1.0.0
 

docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - API_BUILDER_URL=${API_BUILDER_URL}
       - API_BUILDER_SSL_CERT=${API_BUILDER_SSL_CERT}
       - MEMCACHED=${MEMCACHED}
+      - DROP_TRACE_MESSAGE_LEVELS=${DROP_TRACE_MESSAGE_LEVELS}
     ports:
       - 5044:5044
     volumes:

env-sample

@@ -358,10 +358,22 @@ ELASTIC_VERSION=7.14.0
 # PAYLOAD_HANDLING_ENABLED=false
 
 # ----------------------------------------------------------------------------------------------
-# Disables the Elasticsearch setup flows in API-Builder that are used to configure.
+# By default, all generated trace messages are forwarded to Elasticsearch. No matter if DEBUG 
+# or DATA. 
+# If you do not want this, you can configure which trace levels should be dropped. 
+# For example, if you set DEBUG,DATA these two log levels will not be stored in ElasticSearch.
+# Please note that this will of course result in the messages not appearing in the 
+# API Gateway traffic monitor. Also it is not possible to change this parameter afterwards and 
+# expect Trace-Messages are re-processed.
+# Used-By: Logstash for Traffic and General-Trace messages
+# Defaults to all levels are forwarded.
+# DROP_TRACE_MESSAGE_LEVELS=DEBUG,DATA
+
+# ----------------------------------------------------------------------------------------------
+# Disables the setup flows in API-Builder that are used to configure Elasticsearch.
 # If you run more than one API builder, you can set this parameter to true, as it is not
-# necessary. However, it is technically not a problem to have multiple API builders running
-# with setup flow enabled.
+# necessary. However, it is technically not a problem to have multiple API-Builders running
+# with setup flows enabled.
 # Used-By: API-Builder
 # Defaults to false.
 # DISABLE_SETUP_FLOWS=true

helm/templates/elasticApimLogstash/logstash-config.yaml

@@ -24,6 +24,7 @@ data:
   ELASTICSEARCH_HOSTS: {{ required "The value global.elasticsearchHosts is missing." .Values.global.elasticsearchHosts | quote }}
   ELASTICSEARCH_CERT: {{ required "The path to the CA for Logstash to Elasticsearch communication is missing" .Values.global.elasticsearchCa | quote }}
   API_BUILDER_URL: "https://{{ include "apim4elastic.fullname" . }}-apibuilder4elastic:{{ .Values.apibuilder4elastic.port }}"
+  DROP_TRACE_MESSAGE_LEVELS: {{ .Values.logstash.dropTraceMessageLevels | quote }}
   API_BUILDER_SSL_CERT: {{ required "The path to the CA for Logstash to Elasticsearch communication is missing" .Values.global.elasticsearchCa | quote }}
   MEMCACHED: localhost:11211
   LOG_LEVEL: info

helm/values.yaml

@@ -206,6 +206,7 @@ logstash:
   # See parameter LS_JAVA_OPTS in env-sample for more details
   logstashJavaOpts: "-Xmx6g -Xms6g"
   fullnameOverride: "axway-elk-apim4elastic"
+  dropTraceMessageLevels: "DEBUG|DATA"
   # Number of replicas for Logstash
   replicas: 2
   # Injects the environment variables from the ConfigMaps and Secrets into the 

logstash/pipelines/TraceMessagesPipeline.conf

@@ -4,14 +4,21 @@ input { pipeline { address => "TraceMessages" } }
 filter {
   # The logtype has been set by Filebeat to distinct between OpenLog and TraceLog
   if [message] =~ /^#/ {
-    drop { }
+    drop { id => "Drop comments" }
   }
   if [message] =~ /^\s*$/ {
-    drop { }
+    drop { id => "Drop empty lines" }
   }
   grok {
     match => { "message" => "%{LOGLEVEL:level}%{SPACE}(?<loggedDate>\d{2}\/.{3}\/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3})%{SPACE}\[%{WORD:fluff}\:%{WORD:correlationId}\]\s?%{GREEDYDATA:body}" }
   }
+  mutate {
+    add_field => { "[@metadata][dropLevels]" => "${DROP_TRACE_MESSAGE_LEVELS}" }
+  }
+  if( [level] in [@metadata][dropLevels] ) {
+    drop { id => "Log-Level based drop" }
+  }
+
   date {
     # LoggedDate is provided like so 13/Jul/2020:15:26:35.108
     match => [ "loggedDate", "dd/MMM/yyyy:HH:mm:ss.SSS" ]

logstash/test/http/test-tracemessages.json

@@ -62,20 +62,28 @@
     {
       "description": "Trace line causing Logstash to fail and stop responding",
       "input": [
-        "DEBUG   03/Mar/2021:14:22:51.073 [19607:988d3f603f0063aef0c65174]     Event Log config [enabled=true, configFile= NotDefined, outputDir=/opt/axway/apigateway/events, header={\"type\":\"header\", \"logCreationTime\":\"2021-03-03 14:22:51.073\", \"hostname\":\"api-gateway\", \"domainId\":\"fb69e91d-123b-49bc-a964-d76b8a30a810\", \"groupId\":\"group-2\", \"groupName\":\"EXP_GROUP_INT\", \"serviceId\":\"instance-1\", \"serviceName\":\"EXP_API_GW_INT_1\", \"version\":\"v7.7.0-Internal\"}, writeSystemEventFreqSecs=60, customMessageAttributes=[], customSelectorAttributes=[${http.headers[\"X-CorrelationID\"]}, ${http.headers[\"User-Agent\"]}]]"
+        "INFO    03/Mar/2021:14:22:51.073 [19607:988d3f603f0063aef0c65174]     Event Log config [enabled=true, configFile= NotDefined, outputDir=/opt/axway/apigateway/events, header={\"type\":\"header\", \"logCreationTime\":\"2021-03-03 14:22:51.073\", \"hostname\":\"api-gateway\", \"domainId\":\"fb69e91d-123b-49bc-a964-d76b8a30a810\", \"groupId\":\"group-2\", \"groupName\":\"EXP_GROUP_INT\", \"serviceId\":\"instance-1\", \"serviceName\":\"EXP_API_GW_INT_1\", \"version\":\"v7.7.0-Internal\"}, writeSystemEventFreqSecs=60, customMessageAttributes=[], customSelectorAttributes=[${http.headers[\"X-CorrelationID\"]}, ${http.headers[\"User-Agent\"]}]]"
       ],
       "expected": [
         {
           "@timestamp": "2021-03-03T14:22:51.073Z",
           "correlationId": "988d3f603f0063aef0c65174",
-          "level": "DEBUG",
+          "level": "INFO",
           "message": "    Event Log config [enabled=true, configFile= NotDefined, outputDir=/opt/axway/apigateway/events, header={\"type\":\"header\", \"logCreationTime\":\"2021-03-03 14:22:51.073\", \"hostname\":\"api-gateway\", \"domainId\":\"fb69e91d-123b-49bc-a964-d76b8a30a810\", \"groupId\":\"group-2\", \"groupName\":\"EXP_GROUP_INT\", \"serviceId\":\"instance-1\", \"serviceName\":\"EXP_API_GW_INT_1\", \"version\":\"v7.7.0-Internal\"}, writeSystemEventFreqSecs=60, customMessageAttributes=[], customSelectorAttributes=[${http.headers[\"X-CorrelationID\"]}, ${http.headers[\"User-Agent\"]}]]",
           "processInfo": {
             "gatewayName": "API-Gateway 2", 
             "gatewayRegion": "EU"
           }
         }
       ]
+    },
+    {
+      "description": "A trace message on level debug should be dropped",
+      "input": [
+        "DEBUG   03/Mar/2021:14:22:51.073 [19607:988d3f603f0063aef0c65174]     A sample message on level DATA"
+      ],
+      "expected": [
+      ]
     }
   ]
 }