Merge pull request #21 from Axway-API-Management-Plus/scheduled-policies

Adding support to show Scheduled policies in Traffic-Monitor

Author: Chris Wiechmann

.env

@@ -26,14 +26,14 @@ API_BUILDER_URL=http://elk-traffic-monitor-api:8080
 # The Connection/URL to the Admin-Node-Manager. 
 # !!!! IMPORTANT NOTE !!!! 
 # This URL must be reachable/resolveable from within the API-Builder Docker-Container!
-ADMIN_NODE_MANAGER=https://api-env:8090
+ADMIN_NODE_MANAGER=https://172.17.0.1:8090
 
 # ----------------------------------------------------------------------------------------------
 # The Connection/URL to the API-Manager. If not given, the same as Admin-Node-Manager URL with 
 # port 8075 is used.
 # !!!! IMPORTANT NOTE !!!!
 # This URL must be reachable/resolveable from within the API-Builder Docker-Container!
-# API_MANAGER=https://api-env:8075
+# API_MANAGER=https://172.17.0.1:8075
 
 # ----------------------------------------------------------------------------------------------
 # This user is used by API-Builder to lookup APIs & User-Information in API-Manager. 

.github/workflows/api-utils-flow-node.yml

@@ -4,14 +4,7 @@
 
 name: API-Builder API-Utils Flow-Node
 
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-  release: 
-    types: [ published ]
+on: [push, pull_request]
 
 jobs:
   build:

.github/workflows/traffic-monitor-api.yml

@@ -4,14 +4,7 @@
 
 name: Traffic-Monitor API
 
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-  release: 
-    types: [ published ]
+on: [push, pull_request]
 
 jobs:
   build:

docker-compose.yml

@@ -93,7 +93,7 @@ services:
 
   # This is the API-Builder project exposing the API-Gateway Manager REST-API
   elk-traffic-monitor-api:
-    image: cwiechmann/elk-traffic-monitor-api:v0.0.9
+    image: cwiechmann/elk-traffic-monitor-api:latest
     links:
       - elasticsearch1
     environment:

logstash/config/traffic_details_index_template.json

@@ -1,7 +1,6 @@
 {
     "index_patterns": [
-        "apigw-traffic-details-*",
-        "apigw-traffic-errors-*"
+        "apigw-traffic-*"
     ],
     "settings": {
         "number_of_shards": 5, 

logstash/pipeline/pipeline.conf

@@ -21,8 +21,25 @@ filter {
     mutate {
       remove_field => ["host", "timestamp"]
     }
+    # Schedule-Policy event - Most of the properties are NULL
+    if([correlationId]!="000000000000000000000000" and ![circuitPath] and ![transactionElement] and ![transactionSummary][path] and ![transactionSummary][protocol] and ![transactionSummary][protocolSrc]) {
+      mutate { 
+        add_field => { 
+        "[type]" => "scheduledPolicy"
+        "[transactionSummary][serviceContext][service]" => "Scheduled policy"
+        "[transactionSummary][serviceContext][method]" => "N/A"
+        "[transactionElements][leg0][operation]" => "N/A"
+        "[transactionElements][leg0][duration]" => "-1"
+        "[transactionElements][leg0][finalStatus]" => "Pass"
+        "[transactionElements][leg0][protocolInfo][http][uri]" => "N/A"
+        "[transactionElements][leg0][protocolInfo][http][status]" => "901"
+        "[transactionElements][leg0][protocolInfo][http][statusText]" => "OK"
+        "[transactionElements][leg0][protocolInfo][http][method]" => "N/A"
+        } 
+        remove_field => "[transactionSummary][serviceContexts]"
+      }
     # If a transactionElement event is received it must be re-arragended to support up-serts into ES
-    if([transactionElement]) {
+    } else if([transactionElement]) {
       ruby {
         code => "
           leg = event.get('transactionElement');
@@ -134,13 +151,18 @@ output {
   } else if [type] == 'errorEvent' {
     elasticsearch {
       hosts => "elasticsearch1:9200"
-      index => "apigw-traffic-errors-%{+YYYY}"
+      index => "apigw-traffic-errors-%{+YYYY.MM.dd}"
       template => "${HOME}/config/traffic_details_index_template.json"
       template_overwrite => true
       document_id => "%{correlationId}"
       action => "update"
       doc_as_upsert => true
     }
+  } else if [type] == "scheduledPolicy" {
+    elasticsearch {
+      hosts => "elasticsearch1:9200"
+      index => "apigw-traffic-scheduled-%{+YYYY.MM.dd}"
+    }
   } else {
     elasticsearch {
       hosts => "elasticsearch1:9200"

logstash/test/test-openlog-events.json

@@ -313,6 +313,54 @@
             ]
           }
         ]
+      },
+      {
+        "description": "A scheduled policy",
+        "input": [
+          "{\"timestamp\":1598271030003,\"correlationId\":\"36ae435faa3e258ee1092b60\",\"processInfo\":{\"hostname\":\"api-env\",\"domainId\":\"ed992442-c363-4d36-963a-9e6314b0f421\",\"groupId\":\"group-2\",\"groupName\":\"QuickStart Group\",\"serviceId\":\"instance-1\",\"serviceName\":\"QuickStart Server\",\"version\":\"7.7.20200730\"},\"transactionSummary\":{\"path\":null, \"protocol\":null, \"protocolSrc\":null, \"status\":\"success\", \"serviceContexts\":[]}}"
+        ],
+        "expected": [
+          {
+            "@timestamp": "2020-08-24T12:10:30.003Z",
+            "correlationId": "36ae435faa3e258ee1092b60",
+            "logtype": "openlog",
+            "processInfo": {
+              "hostname": "api-env",
+              "domainId": "ed992442-c363-4d36-963a-9e6314b0f421",
+              "groupId": "group-2",
+              "groupName": "QuickStart Group",
+              "serviceId": "instance-1",
+              "serviceName": "QuickStart Server",
+              "version": "7.7.20200730"
+            },
+            "transactionElements": {
+              "leg0": {
+                "operation": "N/A",
+                "finalStatus": "Pass",
+                "duration": "-1",
+                "protocolInfo": {
+                  "http": {
+                    "uri": "N/A",
+                    "status": "901",
+                    "statusText": "OK",
+                    "method": "N/A"
+                  }
+                }
+              }
+            },
+            "transactionSummary": {
+              "path": null,
+              "protocol": null,
+              "protocolSrc": null,
+              "status": "success",
+              "serviceContext": {
+                "method": "N/A",
+                "service": "Scheduled policy"
+              }
+            },
+            "type": "scheduledPolicy"
+          }
+        ]
       }
     ]
   }