Logstash pipeline documentation improved and tests fixed

Author: Chris Wiechmann

logstash/pipelines/OpenTrafficPipeline.conf

@@ -141,6 +141,7 @@ filter {
         if([transactionSummary][serviceContext][service]) {
           mutate { add_field => { "[@metadata][apiName]" => "%{[transactionSummary][serviceContext][service]}" } }
         } else {
+          # Without an API name, only API path is used for the API lookup.
           mutate { add_field => { "[@metadata][apiName]" => "" } }
         }
         # Create a key for the API
@@ -187,9 +188,10 @@ filter {
             # Take over configured custom properties for this API
             copy => { "[apiDetails][customProperties]" => "[transactionSummary][customProperties]" }
           }
-          # Some native APIs (like Healthcheck) don't have a API-Name and Operation (MethodName), so we take it from the looked up API-Details
-          # To avoid that every native API must be configured the local API-Lookup-File, we only do it, when really something has been looked up.
-          # It might also be the case, that the serviceName and operation is set by the Policy. This is then taken over from the leg0 (See below)
+          # The ServiceName and the operation will be taken over only if a name was found at all. This is not the 
+          # case if the API does not exist in the API Manager and has not been configured locally. 
+          # Furthermore, the ServiceName and Operation will not be applied if the ServiceContext has already been set. 
+          # For example with APIs that run through the API Manager and the filter: Set Service Context was used.
           if !([transactionSummary][serviceContext][service] and [apiDetails][name]) {
             mutate {
               copy => { "[apiDetails][name]" => "[transactionSummary][serviceContext][service]" }
@@ -237,7 +239,7 @@ filter {
             remove_field => "[transactionSummary][status]"
           }
           # Place all transactionSummary information directlly into the root, as it makes no sense to have a 
-          # a transactionSummary object within the transactionSummary index.
+          # transactionSummary object within the transactionSummary index.
           ruby {
             code => '
               event.remove("[transactionSummary][path]");
@@ -248,10 +250,10 @@ filter {
             '
           }
         }
-        # Also we need to copy some leg0 information into the summary index
+        # If a transactionElement.leg0 is processed, copy some leg0 information into the summary index
         if [transactionElements][leg0] {
           mutate { 
-            # Before copy most of the properties - Remove some stuff we don't need in the summaryIndex
+            # We don't need all the information in the Summary Index, so some will be removed.
             remove_field => [
               "[transactionElements][leg0][leg]",
               "[transactionElements][leg0][protocolInfo][recvHeader]",
@@ -269,8 +271,8 @@ filter {
               }
             '
           }
-          # A native API may contain a manually set serviceContext (set by Set Service Context filter or WebServiceHandler)
-          # If it exists take it over into the summaryIndex
+          # A native API may contain a serviceName & operation
+          # If it exists, this ServiceName is copied to the TransactionSummary ServiceContext.
           # For managed APIs (API-Manager based) it's the same anyway
           if [transactionElements][leg0][serviceName] {
             mutate {

logstash/test/http/test-opentrafficlog.json

@@ -6,7 +6,7 @@
   "codec": "json_lines",
   "testcases": [
     {
-      "description": "Test transactionElement event with leg0 having a serviceName and operation set",
+      "description": "Test transactionElement event with leg0 having a serviceName and operation set, but not serviceContext",
       "input": [
         "{\"timestamp\":1594673918042,\"correlationId\":\"feca0c5f7f550c81357fdfXX\",\"processInfo\":{\"hostname\":\"ip-172-31-56-150.ec2.internal\",\"domainId\":\"e1ff3aab-3b11-4820-a1be-3763da2beda4\",\"groupId\":\"group-2\",\"groupName\":\"group-a\",\"serviceId\":\"instance-2\",\"serviceName\":\"instance-2\",\"version\":\"7.7.20200530\"},\"transactionElement\":{\"leg\":0,\"duration\":20,\"serviceName\":\"Petstore\",\"operation\":\"getInventory\",\"finalStatus\":\"Pass\",\"protocolInfo\":{\"http\":{\"uri\":\"/petstore/v2/store/inventory\",\"status\":200,\"statusText\":\"OK\",\"method\":\"GET\",\"vhost\":null,\"wafStatus\":0,\"bytesSent\":774,\"bytesReceived\":126,\"remoteName\":\"127.0.0.1\",\"remoteAddr\":\"127.0.0.1\",\"localAddr\":\"127.0.0.1\",\"remotePort\":\"59574\",\"localPort\":\"8065\",\"sslSubject\":null,\"authSubjectId\":\"180b1f32-d72f-40f4-949a-fc3f3f7dec2c\"},\"recvHeader\":\"GET /petstore/v2/store/inventory HTTP/1.0\\r\\nConnection: close\\r\\nHost: localhost\\r\\nKeyId: 6cd55c27-675a-444a-9bc7-ae9a7869184d\\r\\n\\r\\n\",\"sentHeader\":\"HTTP/1.1 200 OK\\r\\nMax-Forwards: 20\\r\\nVia: 1.0 ip-172-31-56-150.ec2.internal (Gateway)\\r\\nConnection: close\\r\\nX-CorrelationID: Id-feca0c5f7f550c81357fdfXX 0\\r\\ncache-control: no-store, no-cache, must-revalidate, proxy-revalidate\\r\\nDate: Mon, 13 Jul 2020 20:58:38 GMT\\r\\netag: W/\\\"52-LxypU3mP2zpjiEr7yZRKlgVdx3c\\\"\\r\\nexpires: 0\\r\\npragma: no-cache\\r\\nrequest-id: c190fa5f-a192-4e57-8eec-c75a6f52b90d\\r\\nresponse-time: 2\\r\\nserver: API Builder/4.28.10\\r\\nstart-time: 1594673918059\\r\\nsurrogate-control: no-store\\r\\nVary: Accept-Encoding\\r\\nx-content-type-options: nosniff\\r\\nx-frame-options: DENY\\r\\nx-xss-protection: 1; mode=block\\r\\ncontent-md5: 43364b0ef391a24f9a0e587356726c0d\\r\\ncontent-type: application/json; charset=utf-8\\r\\n\\r\\n\",\"recvPayload\":null,\"sentPayload\":null}}}"
       ],
@@ -43,11 +43,9 @@
             "serviceId": "instance-2",
             "version": "7.7.20200530"
           },
-          "transactionSummary": {
-            "serviceContext": {
-              "method": "getInventory",
-              "service": "Petstore"
-            }
+          "serviceContext": {
+            "method": "getInventory",
+            "service": "Petstore"
           }
         },
         {