cicd: rewrite bash tests to python (#213)

* add python tests
* add new postgres versions in runner.py
* fix key_and_queries in statements.py with adding info_view

Author: baliukr

mamonsu/lib/runner.py

@@ -224,8 +224,7 @@ def is_any_equal(array):
 #  extract pg version from input
 def define_pg_version(version_args):
     if len(version_args) < 4:
-        if version_args == "15" or version_args == "14" or version_args == "11" or version_args == "12" or version_args == "13" or version_args == "10" \
-                or version_args == "9.6" or version_args == "9.5":
+        if version_args in ["9.5", "9.6", "10", "11", "12", "13", "14", "15", "16", "17"]:
             version_number = version_args[0].split('.')
             for num in version_number:
                 if not num.isdigit():

mamonsu/plugins/pgsql/statements.py

@@ -288,13 +288,16 @@ def keys_and_queries(self, template_zabbix):
                     i + 1))
 
             if Pooler.server_version_greater("14"):
+                info_view = 'pgpro_stats_info'
                 if self.extension == "pg_stat_statements":
-                    for i, item in enumerate(self.Items_pg_14):
-                        keys = item[0].split("[")
-                        result.append(
-                            "{0}[*],$2 $1 -c \"{1}\" | awk -F  '|' '{{print ${2}}}'".format(
-                                "{0}{1}.{2}".format(self.key, keys[0], keys[1][:-1]),
-                                self.query_info.format(metrics=(item[1]), extension_schema=extension_schema),
+                    info_view = 'pg_stat_statements_info'
+                for i, item in enumerate(self.Items_pg_14):
+                    keys = item[0].split("[")
+                    result.append(
+                        "{0}[*],$2 $1 -c \"{1}\" | awk -F  '|' '{{print ${2}}}'".format(
+                            "{0}{1}.{2}".format(self.key, keys[0], keys[1][:-1]),
+                            self.query_info.format(metrics=(item[1]), extension_schema=extension_schema,
+                                                   info_view_name=info_view),
                             i + 1))
             return template_zabbix.key_and_query(result)
         else:

tests/.env

@@ -0,0 +1,29 @@
+WAIT_MAMONSU_TIMEOUT=180
+DEFAULT_HOSTGROUP="Zabbix servers"
+DEFAULT_TEMPLATE="Mamonsu PostgreSQL Linux"
+POSTGRES_VERSION=15
+
+# creds
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=mamonsu_test_db
+
+ZABBIX_ADMIN_USER=Admin
+ZABBIX_ADMIN_PASS=zabbix
+
+# hosts
+ZABBIX_EXT_URL=127.0.0.1:1337
+ZABBIX_INT_URL=zabbix-web:8080
+POSTGRES_EXT_HOST=127.0.0.1
+
+# external ports
+POSTGRES_EXT_PORT=15432
+MAMONSU_AGENT_EXT_PORT=11050
+ZABBIX_SERVER_EXT_PORT=11051
+ZABBIX_WEB_EXT_PORT=1337
+
+# internal ports
+POSTGRES_PORT=5432
+MAMONSU_AGENT_PORT=10050
+ZABBIX_SERVER_PORT=10051
+ZABBIX_WEB_PORT=8080

tests/README.md

@@ -0,0 +1,32 @@
+
+# Mamonsu autotests
+
+Mamonsu testing with different Postgres version, different operation systems(not supported yet). Uses docker-compose to run all services.
+
+
+## Installation
+
+
+```bash
+  pip3 install -e requirement.txt
+```
+
+## Usage/Examples
+
+You can simly run tests with only pytest mark "bash" and it will be ran with Postgres version from env variable POSTGRES_VERSION which is specified in .env file
+
+```bash
+pytest -m bash
+```
+
+You can run tests with different Postgres versions with  POSTGRES_VERSIONS variable
+
+```bash
+POSTGRES_VERSIONS=12,13 pytest -m bash
+```
+
+To run specific test you have to use -k flag with function name
+
+```bash
+POSTGRES_VERSIONS=12,13 pytest -k test_export_zabbix_params
+```

tests/config/__init__.py

@@ -0,0 +1,29 @@
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+
+class Config:
+    def __init__(self, env_path: Path | None = None):
+        self._root_path = Path(__file__).parent.parent
+        load_dotenv(env_path or self._root_path / ".env")
+
+    def __getattr__(self, name: str) -> Any:
+        value = os.getenv(name)
+        if value is None:
+            return None
+        return self._convert_value(value)
+
+    @staticmethod
+    def _convert_value(value: str) -> Any:
+        if value.lower() in ("true", "false"):
+            return value.lower() == "true"
+        try:
+            return int(value)
+        except ValueError:
+            try:
+                return float(value)
+            except ValueError:
+                return value

tests/config/config.py

@@ -0,0 +1,8 @@
+from enum import StrEnum
+
+
+class ContainersEnum(StrEnum):
+    POSTGRES = "mamonsu-pg"
+    MAMONSU = "mamonsu-pg"
+    ZABBIX_WEB = "zabbix-web"
+    ZABBIX_SERVER = "zabbix-server"

tests/config/constants/__init__.py

@@ -0,0 +1,44 @@
+# Стейдж нужен для того, что бы потом скопировать из него энтрипоинт нужной версии, т.к. в --from= нельзя использовать env
+ARG POSTGRES_VERSION=15
+FROM postgres:${POSTGRES_VERSION} AS postgres_base
+
+FROM debian:bookworm-slim AS builder
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    curl \
+    software-properties-common \
+    make \
+    dpkg-dev \
+    debhelper \
+    build-essential \
+    python3-dev \
+    python3-setuptools && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY . /app
+RUN make deb
+
+FROM postgres:${POSTGRES_VERSION}
+
+COPY --from=builder /app/mamonsu*.deb /tmp/
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3 \
+    python3-setuptools \
+    sudo \
+    && rm -rf /var/lib/apt/lists/*
+RUN dpkg -i /tmp/mamonsu*.deb || apt-get install -f -y && \
+    rm /tmp/mamonsu*.deb
+RUN mkdir -p /var/log/mamonsu && \
+    chown postgres:postgres /var/log/mamonsu && \
+    chmod 755 /var/log/mamonsu
+
+COPY --from=postgres_base /usr/local/bin/docker-entrypoint.sh /usr/local/bin/
+COPY ./tests/service-scripts/mamonsu-pg/mamonsu.conf /etc/mamonsu/agent.conf
+COPY ./tests/service-scripts/mamonsu-pg/entrypoint.sh ./tests/service-scripts/mamonsu-pg/init_mamonsu_in_zbx.sh /app/
+
+RUN chmod +x /app/entrypoint.sh /app/init_mamonsu_in_zbx.sh
+
+ENTRYPOINT ["/app/entrypoint.sh"]

tests/config/constants/containers.py

@@ -0,0 +1,68 @@
+services:
+  mamonsu-pg:
+    build:
+      context: .
+      dockerfile: tests/debian.Dockerfile
+      args:
+        POSTGRES_VERSION: ${POSTGRES_VERSION}
+    container_name: mamonsu-pg
+    hostname: mamonsu-pg
+    image: mamonsu-pg
+    ports:
+      - "${MAMONSU_AGENT_EXT_PORT}:${MAMONSU_AGENT_PORT}"
+      - "${POSTGRES_EXT_PORT}:${POSTGRES_PORT}"
+    environment:
+      POSTGRES_VERSION: ${POSTGRES_VERSION}
+      MAMONSU_AGENT_PORT: ${MAMONSU_AGENT_PORT}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: postgres
+      POSTGRES_HOST_AUTH_METHOD: md5
+      ZABBIX_USER: ${ZABBIX_ADMIN_USER}
+      ZABBIX_PASSWD: ${ZABBIX_ADMIN_PASS}
+      ZABBIX_URL: http://${ZABBIX_INT_URL}/
+    restart: no
+
+  zabbix:
+    image: zabbix/zabbix-server-pgsql:6.4.13-ubuntu
+    container_name: zabbix
+    hostname: zabbix
+    environment:
+      - DB_SERVER_HOST=mamonsu-pg
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=zabbix
+      - PGPASSWORD=${POSTGRES_PASSWORD}
+    ports:
+      - "${ZABBIX_SERVER_EXT_PORT}:${ZABBIX_SERVER_PORT}"
+    depends_on:
+      - mamonsu-pg
+
+  zabbix-web:
+    image: zabbix/zabbix-web-nginx-pgsql:6.4.13-ubuntu
+    container_name: zabbix-web
+    hostname: zabbix-web
+    environment:
+      - DB_SERVER_HOST=mamonsu-pg
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=zabbix
+      - ZBX_SERVER_HOST=zabbix-server
+      - ZBX_SERVER_PORT=${ZABBIX_SERVER_PORT}
+      - ZABBIX_ADMIN_USER=Admin
+      - ZABBIX_ADMIN_PASS=zabbix
+    ports:
+      - "${ZABBIX_WEB_EXT_PORT}:${ZABBIX_WEB_PORT}"
+    depends_on:
+      - zabbix
+    healthcheck:
+      test: |
+        curl -fsS "http://localhost:${ZABBIX_WEB_PORT}/api_jsonrpc.php" \
+          -X POST \
+          -H "Content-Type: application/json-rpc" \
+          -d '{"jsonrpc":"2.0","method":"apiinfo.version","id":1,"auth":null,"params":{}}' \
+        | grep -q '"result"' || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 15
+      start_period: 30s