Merge pull request #1 from aliartiza75/add-manifests

[add-manifests] add manifests

Author: aliartiza75

.github/workflows/python-app.yml

@@ -0,0 +1,29 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python application
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with pycodestyle
+      run: |
+        pycodestyle *.py

.gitignore

@@ -0,0 +1 @@
+*.zip

lambda_function.py

@@ -0,0 +1,47 @@
+import os
+import boto3
+import curator
+from requests_aws4auth import AWS4Auth
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+
+ES_HOST = os.environ.get('ES_HOST', 'localhost')
+ES_REGION = os.environ.get('ES_REGION', 'us-west-1')
+ES_INDICES_PREFIX = os.environ.get('ES_INDICES_PREFIX', 'logs')
+ES_INDICES_DATA_RETENTION_THRESHOLD_IN_DAYS = os.environ.get('ES_INDICES_DATA_RETENTION_THRESHOLD_IN_DAYS', '90')
+ES_INDICES_DATE_FORMAT = os.environ.get('ES_INDICES_DATE_FORMAT', '%Y.%m.%d')
+AWS_SERVICE = "es"
+
+
+def lambda_handler(event, context):
+    """
+    Lambda Handler
+    """
+
+    # getting credentials
+    credentials = boto3.Session().get_credentials()
+    awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, ES_REGION, AWS_SERVICE, session_token=credentials.token)
+
+    elasticsearch_client_obj = Elasticsearch(hosts=[{'host': ES_HOST, 'port': 443}],
+                                             http_auth=awsauth,
+                                             use_ssl=True,
+                                             verify_certs=True,
+                                             connection_class=RequestsHttpConnection
+                                            )
+
+    index_list = curator.IndexList(elasticsearch_client_obj)
+
+    # filter the indices based on the prefix (ES_INDICES_PREFIX).
+    index_list.filter_by_regex(kind='prefix',
+                               value=ES_INDICES_PREFIX)
+
+    if index_list.indices:
+        # filter the logs indices that are ES_INDICES_DATA_RETENTION_THRESHOLD_IN_DAYS days old
+        index_list.filter_by_age(source='name',
+                                 direction='older',
+                                 timestring=ES_INDICES_DATE_FORMAT,
+                                 unit='days',
+                                 unit_count=int(ES_INDICES_DATA_RETENTION_THRESHOLD_IN_DAYS))
+
+    # check if indices exist
+    if index_list.indices:
+        curator.DeleteIndices(index_list).do_action()

requirements.txt

@@ -0,0 +1,8 @@
+boto3==1.13.12
+botocore==1.16.12
+pycodestyle==2.5.0
+elasticsearch==7.7.0
+requests==2.23.0
+requests-aws4auth==0.9
+urllib3==1.24.3
+elasticsearch-curator==5.8.1

setup.cfg

@@ -0,0 +1,3 @@
+[pycodestyle]
+ignore = E402,W504,E124,E126,E122
+max-line-length = 150