Added Trace-Messages for local API-Lookup

Author: Chris Wiechmann

README.md

@@ -855,7 +855,7 @@ docker logs logstash -f
 When Logstash is successfully started you should see the following:
 ```
 docker logs logstash
-Pipelines running {:count=>6, :running_pipelines=>[:".monitoring-logstash", :BeatsInput, :Events, :DomainAudit, :TraceMessages, :OpenTraffic], :non_running_pipelines=>[]}
+Pipelines running {:count=>6, :running_pipelines=>[:".monitoring-logstash", :DomainAudit, :Events, :TraceMessages, :BeatsInput, :OpenTraffic], :non_running_pipelines=>[]}
 Successfully started Logstash API endpoint {:port=>9600}
 ```
 If you see the following or similar error message during processing of events the API-Builder Lookup-API cannot be reached. In case, please make sure the environment variable: `API_BUILDER_URL`is set correctly.
@@ -959,9 +959,12 @@ END
 ```
 The example shows in the namespace ignoredAPIs that two entries are contained. So these are not queried again until they have expired.  
 ```
-stats cachedump 1 0
-ITEM index_status:openlog###N/A [4 b; 1609980429 s]
-ITEM index_status:trace###N/A [4 b; 1609980396 s]
+stats cachedump 2 0
+ITEM ignoredAPIs:###Health Check [19 b; 1610535266 s]
+ITEM ignoredAPIs:/healthcheck### [19 b; 1610535266 s]
+ITEM index_status:events###N/A [27 b; 1610538266 s]
+ITEM index_status:trace###N/A [27 b; 1610538266 s]
+ITEM index_status:openlog###N/A [27 b; 1610538266 s]
 END
 ```
 This holds the index creation status used by API-Builder & Logstash to determine if an indicies needs to be created or not.  

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/actions.js

@@ -234,9 +234,11 @@ async function _getAPILocalProxies(params, options) {
 			logger.error(`Error reading API-Lookup file: '${lookupFile}'. Error: ${ex}`);
 			return;
 		}
+		options.logger.info(`Reading API-Details from local file: ${lookupFile}`);
 		localAPIConfig = { ...localProxies };
 		var filenames = await _getGroupRegionFilename(lookupFile, groupId, region);
 		if(filenames.groupFilename) {
+			options.logger.info(`Reading API-Details from local file: ${lookupFile}`);
 			var groupProxies = JSON.parse(fs.readFileSync(filenames.groupFilename), null);
 			localAPIConfig[groupId] = groupProxies;
 		}
@@ -261,11 +263,14 @@ async function _getAPILocalProxies(params, options) {
 }
 
 async function _getLocalProxy(localProxies, apiPath, policyName, options) {
-	if(localProxies == undefined) return;
+	if(localProxies == undefined) {
+		options.logger.warn(`No configuration found in file.`);
+		return;
+	}
 	var foundProxy;
 	// If a policy is given, it is used separately for the lookup
 	if(policyName != undefined && policyName != "") {
-		options.logger.debug(`Looking up information based on policy name: ${policyName}`);
+		options.logger.info(`Looking up information based on policy name: ${policyName}`);
 		if(localProxies[`Policy: ${policyName}`]) {
 			foundProxy =  localProxies[`Policy: ${policyName}`];
 		}
@@ -284,8 +289,10 @@ async function _getLocalProxy(localProxies, apiPath, policyName, options) {
 		};
 		// Perhaps, we have direct hit with the API-Path
 		if(localProxies[apiPath]) {
+			options.logger.info(`API-Details found based on API-Path: ${apiPath}`);
 			foundProxy = localProxies[apiPath];
 		} else {
+			options.logger.info(`Try to find API-Details starting with: ${apiPath}`);
 			// Iterate over all configured API-Proxies
 			for (const [key, val] of Object.entries(localProxies)) { 
 				if(apiPath.startsWith(key)) {
@@ -295,9 +302,13 @@ async function _getLocalProxy(localProxies, apiPath, policyName, options) {
 		}
 	}
 	// If we don't have a match return nothing
-	if(foundProxy==undefined) return;
+	if(foundProxy==undefined) {
+		options.logger.warn(`No API-Details found for API-Path: ${apiPath}`);
+		return;
+	}
 	// Take over the configuration, preserve the default values
 	proxy = {...proxy, ...foundProxy};
+	options.logger.warn(`Found API-Details for API-Path: ${JSON.stringify(apiPath)}`);
 	var proxies = [];
 	proxy.path = apiPath; // Copy the path, as it's normally returned by the API-Manager and used for caching
 	proxies.push(proxy);

logstash/pipelines/BeatsInputPipeline.conf

@@ -31,6 +31,9 @@ filter {
   }
   # If the version has just checked, add it to the cache to avoid further lookups
   if([@metadata][updateVersionCheckCache]=="true") {
+    ruby {
+      code => 'logger.info("Adding version check status to cache: ", "value" => event.get("[versionStatus]") );'
+    }
     memcached {
       hosts => "${MEMCACHED}"
       namespace => "version_status"
@@ -90,7 +93,7 @@ filter {
   # If the API-Builder create index API has been called, create a cache entry to avoid further unneeded calls
   if([@metadata][updateCache]=="true") {
     ruby {
-      code => 'logger.info("Adding index status to cache: ", "value" => event.get("[indicesStatus]"), "logtype" => event.get("[logtype]"), "region" => event.get("[gatewayRegion]"));'
+      code => 'logger.info("Adding index status to cache: ", "value" => event.get("[indicesStatus]"), "logtype" => event.get("[logtype]"), "region" => event.get("[gatewayRegion]") );'
     }
     memcached {
       hosts => "${MEMCACHED}"

logstash/pipelines/OpenTrafficPipeline.conf

@@ -59,10 +59,16 @@ filter {
   }
   # Drop the document if it should be ignored
   if ([isIgnoreAPI][ignore]) {
+    ruby {
+      code => 'logger.debug("Drop API/event as it should be ignored: ", "isIgnoreAPI" => event.get("[isIgnoreAPI]"), "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]") );'
+    }
     drop {}
   }
   # If the PolicyName or API-Path has been looked up right now add it to the cache
   if([@metadata][updateIsIgnoreCache]=="true") {
+    ruby {
+      code => 'logger.info("Adding ignore status to cache: ", "isIgnoreAPI" => event.get("[isIgnoreAPI]"), "ignoreLookupCacheKey" => event.get("[@metadata][ignoreLookupCacheKey]") );'
+    }
     memcached {
       hosts => "${MEMCACHED}"
       namespace => "ignoredAPIs"
@@ -185,6 +191,9 @@ filter {
           }
           # If the API has been looked up add it to the cache
           if([@metadata][updateAPICache]=="true") {
+            ruby {
+              code => 'logger.info("Adding API-Lookup details to cache: ", "apiDetails" => event.get("[apiDetails]"), "apiCacheKey" => event.get("[apiCacheKey]") );'
+            }
             memcached {
               hosts => "${MEMCACHED}"
               namespace => "api_details"