Merge pull request #1 from cwiechmann/master

Majar refactoring

Author: Chris Wiechmann

.env

@@ -1,2 +1,9 @@
+# Filebeat will mount that folder into the Filebeat Docker-Container to have access to the files
 APIGATEWAY_LOGS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/logs/opentraffic
 APIGATEWAY_TRACES_FOLDER=/home/localuser/Axway-x.y.z/apigateway/groups/group-1/instance-1/trace
+
+# This variable is used by the API-Builder project to locate the Elasticsearch instance
+# Using the default docker-compose.yaml this setting is sufficient
+# When running the API-Builder on a different host,  this variable needs to be made available to the 
+# container.
+ELASTIC_NODE=http://elasticsearch1:9200

.github/workflows/traffic-monitor-api.yml

@@ -0,0 +1,68 @@
+# This workflow is validating the API-Builder exposed Traffic-Monitor API. 
+# For that an Elasticsearch instance is started, test-data inserted and the 
+# API-Builder Traffic-Monitor API is executed with all possible parameters.
+
+name: Test Traffic-Monitor API
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  release: 
+    types: [ published ]
+
+jobs:
+  build:
+    env:
+      workingDirectory: 'elk-traffic-monitor-api'
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Starting Elasticsearch instance
+      uses: nyaruka/elasticsearch-action@v1
+      with:
+        elastic version: '7.6.1' 
+    - name: Sleep 30 seconds to make sure ES is ready
+      uses: jakejarvis/wait-action@master
+      with:
+        time: '30s'
+    - name: Run npm ci, npm test
+      working-directory: ${{env.workingDirectory}}
+      env:
+        ELASTIC_NODE: 'http://localhost:9200'
+        LOG_LEVEL: INFO
+        CI: true
+      run: | 
+        npm ci
+        npm run build --if-present
+        npm test
+  publish:
+    if: github.event.action == 'published'
+    env:
+      workingDirectory: 'elk-traffic-monitor-api'
+    needs: build
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Publish Docker image
+      uses: elgohr/Publish-Docker-Github-Action@2.13
+      with:
+        name: cwiechmann/elk-traffic-monitor-api
+        workdir: ${{env.workingDirectory}}
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        snapshot: true
+

README.md

@@ -1,25 +1,121 @@
 # API-Management Traffic-Monitor based ELK stack
 
+When having many API-Gateway instances with millions of requests the API-Gateway Traffic Monitor can become slow. The purpose of this project is to solve that performance issue and get other benefits by using a standard external datastore: Elasticsearch.  
+
+The overall architecture this project provides looks like this:  
+![Architecture][img1]   
+
+### How it works  
+Each API-Gateway instance is writing, [if configured](#enable-open-traffic-event-log), Open-Traffic Event-Log-Files, which are streamed by [Filebeat](https://www.elastic.co/beats/filebeat) into a Logstash-Instance. [Logstash](https://www.elastic.co/logstash) performs data pre-processing, combines different events and finally forwards the document into an [Elasticsearch](https://www.elastic.co/elasticsearch) cluster.  
+
+Once the data is indexed by Elasticsearch it can be used by different clients. This process allows almost realtime monitoring of incoming requests. It takes around 5 seconds until a request is available in Elasticsearch.
+
+## Option 1 - Using the existing Traffic-Monitor
+One option is to use the existing API-Gateway Traffic-Monitor. That means, you use the same tooling as of today, but the underlying implementation of the Traffic-Monitor API is now pointing to Elasticsearch instead of the internal OPSDB hosted by each API-Gateway instance. This improves performance damatically, as Elasticsearch can scale across multiple machines if required and other dashboards can be created for instance with Kibana.  
+The glue between Elasticsearch and the API-Gateway Traffic-Monitor is an API-Builder project, that is exposing the same Traffic-Monitor API, but it is implemented using Elasticsearch instead of the OPSDB. The API-Builder is available as a ready to use Docker-Image and preconfigured in the docker-compose file.  
+Finally, the Admin-Node-Manager has to be configured to use the API-Builder API instead of the internal implementation.
+
+API-Builder status:  
+![Test Traffic-Monitor API](https://github.com/cwiechmann/apigateway-openlogging-elk/workflows/Test%20Traffic-Monitor%20API/badge.svg)
+
+## Option 2 - Logspector
+The Logspecotr is a new separated user-interface with very basic set of functionilties. As part of the project the Logspector is activated by default when using `docker-compose up -d`. If you don't wanna use it, it can be disabled by commenting out the following lines in the docker-compose.yml file:
+```yaml
+  nginx:
+    image: nginx:1.17.6
+    ports:
+      - 8888:90
+    volumes:
+      - ${PWD}/nginx/www:/usr/share/nginx/html
+      - ${PWD}/nginx/conf:/etc/nginx
+    depends_on:
+      - elasticsearch1
+    networks:
+      - elastic
+      - ingress
+```
+The Log-Inspector is accessible on the following URL: `http://hostname-to-your-docker-machine:8888/logspector.html`
+
+![Log-Spector][img5]  
+
+
 ## Prerequisites
+For a simple deployment the prerequisites are very simple as all services can be started as a Docker-Container. In order to start all components in PoC-Like-Mode you just need:
+
+1. A Docker engine
+2. docker-compose installed
+3. An API-Management Version >7.7-20200130  
+   - Versin 7.7-20200130 is required due to some Dateform changes in the Open-Traffic-Format. With older versions of the API-Gateway you will get an error in Logstash processing.
 
-1. docker
-2. docker-compose
-3. API-Management Version >7.7-20200130
+Using the provided docker-compose is good to play with, however this approach is not recommended for production environments. Depending the load a dedicated machine (node) for Elasticsearch is recommended. The default configuration is prepared to scale up to five Elasticsearch nodes, which can handle millions of requests. To run Logstash and the API-Builder service a Docker-Orchestration framework is recommended as you get monitoring, self-healing, elasticity.
+
+## Installation / Configuration
+To run the components in a PoC-Like mode, the recommended way is to clone this project onto a machine having docker and docker-compose installed. Also this machine must have file-based access to the running API-Gateway instance, as the Filebeat docker container will mount the open-traffic folder into the docker-container.
+
+`git clone https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk.git`  
+
+This creates a local copy of the repository and you can start from there.
+
+### Enable Open-Traffic Event Log
+Obviously you have to enable Open-Traffic-Event log for your API-Gateway instances. [Read here][1] how to enable the Open-Traffic Event-Log.  
+After this configuration has been done, Open-Traffic log-files will created by default in this location: `apigateway/logs/opentraffic`. This location becomes relevant in the next step, when configuring Filebeat.
+
+### Configure the Admin-Node-Manager
+This step is required if you would like to use the existing Traffic-Monitor in combination Elasticsearch.  
+The Admin-Node-Manager (listening by default on port 8090) is responsible to server the API-Manager Traffic-Monitor and needs to be configured to use the API-Builder API instead.  
+For the following steps, please open the Admin-Node-Manager configuration in Policy-Studio. You can [here](https://docs.axway.com/bundle/axway-open-docs/page/docs/apim_administration/apigtw_admin/general_rbac_ad_ldap/index.html#use-the-ldap-policy-to-protect-management-services) how to do that.  
+- Create a new policy called: `Use Elasticsearch API`
+- Configure this policy like so:  
+  ![use ES API][img3]  
+    - The `Compare Attribute` filter checks if the requested API is already handled by the API-Builder project.  
+    _As of today, only the Traffic-Overview is handled by the API-Builder. This will be changed soon._
+    - Add the following: `http.request.path` is `/api/router/service/instance-1/ops/search`  
+    _The list of requests will be extended once the API-Builder project can serve more (e.g. the Request-Detail view)_
+    ![Is API Managed][img6] 
+    - Adjust the URL of the Connect to URL filter to your running API-Builder docker container and port (default is 8889).  
+    ![Connect to ES API][img7]  
+- Insert the created policy as a callback policy into the main policy: `Protect Management Interfaces` like so:  
+  ![Use Callback][img4]  
+  
+After you have saved back the configuration to the Admin-Node-Manager and restarted it, the Admin-Node-Manager will use the API-Builder API (Elasticsearch) to serve the specified request-types.  
+
+### Setup filebeat
+:exclamation: __This is an important step, as otherwise Filebeat will not see and send any Open-Traffic Event data!__  
+Before starting the container using docker-compose, make sure to setup the paths in the .env file to your running API-Gateway instance. This configuration is used to mount the Open-Traffic-Folder into Filebeat container.
+```
+APIGATEWAY_LOGS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/logs/opentraffic
+APIGATEWAY_TRACES_FOLDER=/home/localuser/Axway-x.y.z/apigateway/groups/group-1/instance-1/trace
+```
+
+### Setup API-Builder
+As the API-Builder container needs to communicate with Elasticsearch it needs to know where Elasticsearch is running. Use this environment variable to configure it:
+```
+ELASTIC_NODE=http://elasticsearch1:9200
+```
+Please note, when using the default docker-compose.yaml the default setting is sufficient, as it's using the internal Docker-Network `elastic`.
 
 ### Update vm.max_map_count kernel setting to at least 262144
 
 See https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites
 
-## Start local elasticsearch cluster
-
-1.  Edit the .env file and configure both APIGATEWAY_LOGS_FOLDER and APIGATEWAY_TRACES_FOLDER environment variables to point to your open traffic and trace folders respectively.
-2.  Bring the cluster up using docker-compose:
+###  Start local elasticsearch cluster
+Bring the cluster up using docker-compose:
 ````
 docker-compose up -d
 ````
+Of course, the components can also run on different machines or on a Docker-Orchestration framework such as Kubernetes.
 
 ## Stop cluster
 ````
 docker-compose down
 ````
 
+[img1]: imgs/component-overview.png
+[img2]: imgs/node-manager-policies.png
+[img3]: imgs/node-manager-use-es-api.png
+[img4]: imgs/node-manager-policies-use-elasticsearch-api.png
+[img5]: imgs/Logspector.png
+[img6]: imgs/IsmanagedbyElasticsearchAPI.png
+[img7]: imgs/connect-to-elasticsearch-api.png
+
+[1]: https://docs.axway.com/bundle/axway-open-docs/page/docs/apim_administration/apigtw_admin/admin_open_logging/index.html#configure-open-traffic-event-logging

configs/logstash.conf

@@ -185,6 +185,8 @@ output {
     elasticsearch {
       hosts => "elasticsearch1:9200"
       index => "logstash-openlog"
+      template => "${HOME}/pipeline/openlog_index_template.json"
+      template_overwrite => true
     }
   } else {
     elasticsearch {

configs/openlog_index_template.json

@@ -0,0 +1,18 @@
+{
+    "index_patterns": [
+        "*openlog*"
+    ],
+    "settings": {
+        "number_of_shards": 5
+    },
+    "mappings": {
+        "properties": {
+            "processInfo.serviceId": {
+                "type": "keyword"
+            },
+            "transactionElements.leg0.protocolInfo.http.remoteName": {
+                "type": "keyword"
+            }
+        }
+    }
+}

docker-compose.yml

@@ -1,5 +1,6 @@
 version: '3.7'
 services:
+  # The core component
   elasticsearch1:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
     container_name: elasticsearch1
@@ -26,7 +27,7 @@ services:
       - 9200:9200
       - 9300:9300
 
-
+  # This is optional, but good to have to perform manual queries and create custom dashboards
   kibana:
     image: docker.elastic.co/kibana/kibana:7.4.0
     container_name: kibana
@@ -49,7 +50,7 @@ services:
       - elastic
       - ingress
 
-
+  # Supposed to run side-by-side with the API-Gateway to watch the Open-Traffic Event files and send event to Logstash
   filebeat:
     image: docker.elastic.co/beats/filebeat:7.4.0
     command: --strict.perms=false
@@ -67,7 +68,7 @@ services:
     networks:
       - elastic
 
-
+  # Is receiving events from Filebeat and does pre-processing
   logstash:
     image: docker.elastic.co/logstash/logstash:7.4.0
     links:
@@ -78,14 +79,29 @@ services:
     ports:
       - 5044:5044
     volumes:
-      - ${PWD}/configs/logstash.conf:/usr/share/logstash/pipeline/default.conf
-    command: logstash --path.config /usr/share/logstash/pipeline/default.conf --pipeline.batch.size 20 --pipeline.workers 1 --log.level info
+      - ${PWD}/configs/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
+      - ${PWD}/configs/openlog_index_template.json:/usr/share/logstash/pipeline/openlog_index_template.json
+    command: logstash --path.config /usr/share/logstash/pipeline/logstash.conf --pipeline.batch.size 20 --pipeline.workers 1 --log.level info
     depends_on:
       - elasticsearch1
     networks:
       - elastic
 
+  # This is the API-Builder project exposing the API-Gateway Manager REST-API
+  elk-traffic-monitor-api:
+    image: cwiechmann/elk-traffic-monitor-api:latest
+    links:
+      - elasticsearch1
+    environment:
+      - ELASTIC_NODE=http://elasticsearch1:9200
+    ports:
+      - 8889:8080
+    depends_on:
+      - elasticsearch1
+    networks:
+      - elastic
 
+  # This is the Logspector Web-Application
   nginx:
     image: nginx:1.17.6
     ports:

elk-traffic-monitor-api/.dockerignore

@@ -0,0 +1,7 @@
+conf/local.js
+conf/*.local.js
+coverage
+.git
+node_modules
+.nyc_output
+conf/.env

elk-traffic-monitor-api/.gitignore

@@ -0,0 +1,4 @@
+node_modules
+logs
+coverage
+nyc_output

elk-traffic-monitor-api/Dockerfile

@@ -0,0 +1,30 @@
+# See the README.md for usage and configuration info
+
+# This line defines which node.js Docker image to leverage
+# Available versions are described at https://hub.docker.com/_/node/
+FROM node:8-alpine
+
+# Sets the default working directory to /app which is where we copy the service files to.
+WORKDIR /app
+
+# TODO: for security purposes, you should update this Dockerfile to specify your own target user/group
+# -S stands for '--system'
+# -G stands for group
+# -R changes the ownership rights of a file recursively
+RUN addgroup -S axway-group && adduser -S axway-user -G axway-group && \
+	chown -R axway-user:axway-group /app
+
+# Set non-root user
+USER axway-user
+
+# Denotes to copy all files in the service to 'app' folder in the container
+COPY --chown=axway-user:axway-group . /app
+
+# Install service dependencies relevant for production builds skipping all development dependencies.
+RUN npm install --production --no-optional
+
+# check every 5s to ensure this service is healthy
+HEALTHCHECK --interval=5s --start-period=10s --timeout=5s --retries=5 CMD node healthcheck.js
+
+# Starts the service
+CMD ["node", "."]