diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml new file mode 100644 index 0000000000..bad8507ce6 --- /dev/null +++ b/.github/workflows/ci-env-images.yaml @@ -0,0 +1,81 @@ +name: Build and Publish CI Environment Images + +on: + pull_request: + branches: + - develop + paths: + - 'tests/container_images/docker/ubuntu_24/Dockerfile' + - 'tests/container_images/docker/redhat_9/Dockerfile' + + +jobs: + build-and-push-ci-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Required to push to GHCR + + strategy: + fail-fast: false # Set to false to build all images even if one fails + matrix: + os_variant: [ubuntu, rhel] + include: + - os_variant: ubuntu + dockerfile_path: tests/container_images/docker/ubuntu_24/Dockerfile + image_name_suffix: ci-env-ubuntu + # Build args specific to Ubuntu + ansible_version: "6.1.0" + ansible_core_version: "2.13.13" + poetry_installer_version: "1.5.1" + - os_variant: rhel + dockerfile_path: tests/container_images/docker/redhat_9/Dockerfile + image_name_suffix: ci-env-rhel + # Build args specific to RHEL + ansible_version: "6.1.0" + ansible_core_version: "2.13.13" + poetry_installer_version: "1.5.1" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USER }} # Your Docker Hub username + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }} + tags: | + type=raw,value=latest + + - name: Build and push Docker image for ${{ matrix.os_variant }} + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile_path }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + secrets: | + pipconf=${{ secrets.PIP_CONF_CONTENT }} # Pass pip.conf content as a secret + build-args: | + ANSIBLE_VERSION=${{ matrix.ansible_version }} + ANSIBLE_CORE_VERSION=${{ matrix.ansible_core_version }} + POETRY_INSTALLER_VERSION=${{ matrix.poetry_installer_version }} \ No newline at end of file diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index d0d1c137b5..90621d9873 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,14 +178,13 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,42 +199,64 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + + - name: Debug + run: | + whoami + uname -a + cat /etc/os-release + cat ansible/inventory/inventory.yaml + cat ansible/resources/env_file + docker ps + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml + - name: Debug 2 + run: | + whoami + uname -a + docker ps + systemctl status sc4s + docker images - name: Run tests run: | pip3 install poetry @@ -244,11 +265,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-ipv4-name-cache: runs-on: ubuntu-latest @@ -462,4 +489,4 @@ jobs: @google/semantic-release-replace-plugin@1.2.0 conventional-changelog-conventionalcommits@6.1.0 env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} \ No newline at end of file diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index 655405449b..acf6ad215a 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -4,6 +4,10 @@ vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: + - name: Check if 'docker' command exists on target (if not using docker connection) + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true - name: Docker installation role include_role: name: install_docker diff --git a/ansible/resources/env_file b/ansible/resources/env_file index 9f73a6d46c..3eb4275e4d 100644 --- a/ansible/resources/env_file +++ b/ansible/resources/env_file @@ -1,4 +1,4 @@ -SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://xxx.xxx.xxx.xxx:8088 +SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://xxx.xxx.xxx.xxx:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxxxxxxxxxxxx #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no \ No newline at end of file diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..d3d434a774 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -23,10 +23,16 @@ repo: deb https://download.docker.com/linux/ubuntu focal stable state: present +- name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true + - name: Install docker-ce apt: name: docker-ce state: latest + when: docker_command_check.rc != 0 - name: Install Docker Module for Python pip: diff --git a/ansible/roles/install_docker/tasks/install_docker_rhel.yml b/ansible/roles/install_docker/tasks/install_docker_rhel.yml index 48a9f6e9bb..d3938202e0 100644 --- a/ansible/roles/install_docker/tasks/install_docker_rhel.yml +++ b/ansible/roles/install_docker/tasks/install_docker_rhel.yml @@ -28,11 +28,11 @@ state: latest update_cache: true -- name: Install Docker Module for Python - pip: - name: - - docker - - jsondiff +# - name: Install Docker Module for Python +# pip: +# name: +# - docker +# - jsondiff - name: Start Docker ansible.builtin.service: diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 50462fac88..85ad163724 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/docker_sc4s.service" + src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 07632d8eaa..f74fe85807 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/podman_sc4s.service" + src: "../../../ansible/resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/docs/gettingstarted/ansible-docker-podman.md b/docs/gettingstarted/ansible-docker-podman.md index 3f890ffa7a..1de2f6fd6b 100644 --- a/docs/gettingstarted/ansible-docker-podman.md +++ b/docs/gettingstarted/ansible-docker-podman.md @@ -23,16 +23,16 @@ docker exec -it ansible_sc4s /bin/bash * To authenticate with username and password: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/podman.yml ``` * To authenticate using a key pair: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/podman.yml ``` # Step 3: Validate your configuration diff --git a/docs/gettingstarted/ansible-docker-swarm.md b/docs/gettingstarted/ansible-docker-swarm.md index a34f4c1336..39417f3d96 100644 --- a/docs/gettingstarted/ansible-docker-swarm.md +++ b/docs/gettingstarted/ansible-docker-swarm.md @@ -51,7 +51,7 @@ ansible-playbook -i path/to/inventory_swarm.yaml -u --key-file /etc/syslog-ng/VERSION +RUN echo "$VERSION">/etc/syslog-ng/VERSION ENTRYPOINT ["/entrypoint.sh"] diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index f55a657882..61301205f9 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py index 13bb14603d..76081f7f80 100755 --- a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py @@ -36,7 +36,7 @@ def normalize_env_variable_input(env_variable: str): modev = os.environ.get(f"SC4S_DEST_SPLUNK_HEC_{r}_MODE", "GLOBAL") if ( r == "DEFAULT" - and not os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") == "" + and os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") != "" ): if os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "yes").lower() in [ "true", @@ -68,9 +68,8 @@ def normalize_env_variable_input(env_variable: str): if r != "": modev = os.environ.get(f"SC4S_DEST_{t}_{r}_MODE", "GLOBAL") filter = os.environ.get(f"SC4S_DEST_{t}_{r}_FILTER", "") - if filter == "": - if t == "BSD": - filter = '"${MSG}" ne ""' + if filter == "" and t == "BSD": + filter = '"${MSG}" ne ""' if modev.upper() in ("GLOBAL", "SELECT"): global_dests[r] = { "destination": f"d_{t.lower()}_{r.lower()}", diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py index fdf790b0df..1d48a8a6b1 100755 --- a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py @@ -53,8 +53,6 @@ def normalize_env_variable_input(env_variable: str): if len(port_parts) == 2 or len(port_parts) == 3: vendor = port_parts[0].lower() product = port_parts[1].lower() - else: - pass outputText = tm.render( vendor=vendor, @@ -131,8 +129,8 @@ def normalize_env_variable_input(env_variable: str): "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", ), ebpf_no_sockets=int(os.getenv("SC4S_EBPF_NO_SOCKETS", 4)), - enable_parallelize=normalize_env_variable_input(f"SC4S_ENABLE_PARALLELIZE"), - parallelize_no_partitions=int(os.getenv(f"SC4S_PARALLELIZE_NO_PARTITION", 4)), + enable_parallelize=normalize_env_variable_input("SC4S_ENABLE_PARALLELIZE"), + parallelize_no_partitions=int(os.getenv("SC4S_PARALLELIZE_NO_PARTITION", 4)), set_source_sc4s=normalize_env_variable_input("SC4S_SET_SOURCE_AS_SC4S"), ) print(outputText) diff --git a/package/enterprise/etc/pylib/parser_cef.py b/package/enterprise/etc/pylib/parser_cef.py index 7d0673fd45..13e774960c 100644 --- a/package/enterprise/etc/pylib/parser_cef.py +++ b/package/enterprise/etc/pylib/parser_cef.py @@ -21,14 +21,13 @@ def parse(self, log_message): try: data = log_message.get_as_str(".metadata.cef.ext", "") - rpairs = re.findall(r"([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)", data) + rpairs = re.findall(r"([^=\s]+)=((?:\\=|[^=])+)(?:\s|$)", data) pairs = {} keys = [] for p in rpairs: pairs[p[0]] = p[1] keys.append(p[0]) - cleanpairs = {} for k in keys: if k.endswith("Label"): vk = k.rstrip("Label") @@ -50,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_fix_dns.py b/package/enterprise/etc/pylib/parser_fix_dns.py index de3bbd7d15..a35e30dbea 100644 --- a/package/enterprise/etc/pylib/parser_fix_dns.py +++ b/package/enterprise/etc/pylib/parser_fix_dns.py @@ -3,7 +3,6 @@ resolves IP to hostname value pair names are hard-coded """ -import re import socket try: @@ -25,7 +24,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + hostname, _, _ = socket.gethostbyaddr(ipaddr) if hostname == ipaddr: return False @@ -51,7 +50,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + fqdn, _, _ = socket.gethostbyaddr(ipaddr) if fqdn == ipaddr: return False diff --git a/package/enterprise/etc/pylib/parser_kvqf.py b/package/enterprise/etc/pylib/parser_kvqf.py index c5596c55f0..b174b7b6b0 100644 --- a/package/enterprise/etc/pylib/parser_kvqf.py +++ b/package/enterprise/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for matchNum, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/enterprise/etc/pylib/parser_leef.py b/package/enterprise/etc/pylib/parser_leef.py index 1279701314..5c5010cb2f 100644 --- a/package/enterprise/etc/pylib/parser_leef.py +++ b/package/enterprise/etc/pylib/parser_leef.py @@ -1,7 +1,5 @@ import re import binascii -import sys -import traceback try: import syslogng @@ -17,45 +15,64 @@ def init(self, options): self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))" self.logger = syslogng.Logger() return True + + def parse_message_from_pair(self, pair, log_message): + f, v = pair.split("=", 1) + if f == "devTime": + log_message[".leef." + f] = re.sub( + self.regex, "", v, 0, re.MULTILINE + ) + else: + log_message[".leef." + f] = v - def parse(self, log_message): + def parse_v1(self, log_message, event, structure, separator): + pairs = event.split(separator) + if len(pairs) < 4: + separator = "|" + pairs = structure[5:] + event = "\t".join(pairs) + log_message[".leef.event"] = event + return event, pairs, separator + + def parse_v2(self, event, structure, separator): + # V2 messages should always provide the sep but some fail do comply + # with the format spec if they don't assume tab + if len(structure) == 6 or not structure[5]: + pairs = event.split(separator) + else: + separator = structure[5] + if separator.startswith("0"): + separator = separator[1:] + pairs = event.split(separator) + return event, pairs, separator + def parse(self, log_message): try: msg = log_message.get_as_str("MESSAGE", "") # All LEEF message are | separated super structures structure = msg.split("|") - # Indexed fields for Splunk + # Indexed fields for Splunk log_message[".metadata.leef.version"] = structure[0][5:] log_message[".metadata.leef.vendor"] = structure[1] log_message[".metadata.leef.product"] = structure[2] log_message[".metadata.leef.product_version"] = structure[3] log_message[".metadata.leef.EventID"] = structure[4] + # We just want the event field event = structure[len(structure) - 1] log_message[".leef.event"] = event + + separator = "\t" + pairs = [] + # V1 will always use tab if structure[0][5:].startswith("1"): - separator = "\t" lv = "1" - pairs = event.split(separator) - if len(pairs) < 4: - separator = "|" - pairs = structure[5:] - event = "\t".join(pairs) - log_message[".leef.event"] = event + event, pairs, separator = self.parse_v1(log_message, event, structure, separator) else: lv = "2" - # V2 messages should always provide the sep but some fail do comply - # with the format spec if they don't assume tab - if len(structure) == 6 or not structure[5]: - separator = "\t" - pairs = event.split(separator) - else: - separator = structure[5] - if separator.startswith("0"): - separator = separator[1:] - pairs = event.split(separator) + event, pairs, separator = self.parse_v2(event, structure, separator) if separator.startswith("x"): hex_sep = f"0{separator.lower()}" @@ -70,15 +87,9 @@ def parse(self, log_message): log_message["fields.sc4s_product"] = structure[2] for p in pairs: - f, v = p.split("=", 1) - if f == "devTime": - log_message[".leef." + f] = re.sub( - self.regex, "", v, 0, re.MULTILINE - ) - else: - log_message[".leef." + f] = v + self.parse_message_from_pair(p, log_message) except Exception as e: log_message[".metadata.leef.exception"] = str(e) # return True, other way message is dropped - return True \ No newline at end of file + return True diff --git a/package/enterprise/etc/pylib/parser_stealthbits.py b/package/enterprise/etc/pylib/parser_stealthbits.py index bdf0369816..89445d5119 100644 --- a/package/enterprise/etc/pylib/parser_stealthbits.py +++ b/package/enterprise/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: @@ -10,6 +9,7 @@ class LogParser: regex = r"^(.*[\.\!\?])?(.*:.*)" +alert_text_key = ".values.AlertText" class alerttext_kv(LogParser): @@ -17,13 +17,13 @@ def init(self, options): return True def parse(self, log_message): - match = re.search(regex, log_message.get_as_str(".values.AlertText", "")) + match = re.search(regex, log_message.get_as_str(alert_text_key, "")) if match: - log_message[".values.AlertText"] = match.groups()[0] + log_message[alert_text_key] = match.groups()[0] text = match.groups()[1] else: - text = log_message.get_as_str(".values.AlertText", "") - log_message[".values.AlertText"] = "" + text = log_message.get_as_str(alert_text_key, "") + log_message[alert_text_key] = "" pairs = text.split("; ") diff --git a/package/enterprise/etc/pylib/parser_vps_cache.py b/package/enterprise/etc/pylib/parser_vps_cache.py index 4c8cf21250..d545620158 100644 --- a/package/enterprise/etc/pylib/parser_vps_cache.py +++ b/package/enterprise/etc/pylib/parser_vps_cache.py @@ -95,7 +95,3 @@ def send(self, log_message): def flush(self): self.db.commit() return True - - -if __name__ == "__main__": - pass diff --git a/package/etc/pylib/parser_cef.py b/package/etc/pylib/parser_cef.py index 28fd68498e..32dede7e12 100644 --- a/package/etc/pylib/parser_cef.py +++ b/package/etc/pylib/parser_cef.py @@ -49,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True diff --git a/package/etc/pylib/parser_kvqf.py b/package/etc/pylib/parser_kvqf.py index 99206af8d1..b7d3253681 100644 --- a/package/etc/pylib/parser_kvqf.py +++ b/package/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for match_num, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/etc/pylib/parser_stealthbits.py b/package/etc/pylib/parser_stealthbits.py index 07a3c87f10..99800d479f 100644 --- a/package/etc/pylib/parser_stealthbits.py +++ b/package/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: diff --git a/tests/container_images/docker/centos_7/Dockerfile b/tests/container_images/docker/centos_7/Dockerfile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/container_images/docker/redhat_9/Dockerfile b/tests/container_images/docker/redhat_9/Dockerfile new file mode 100644 index 0000000000..a8adf51c3c --- /dev/null +++ b/tests/container_images/docker/redhat_9/Dockerfile @@ -0,0 +1,99 @@ +# syntax=docker/dockerfile:1.4 +FROM redhat/ubi9 + +# Use bash with pipefail for robust scripting +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Set environment variables for Poetry +ENV PATH="/root/.local/bin:$PATH" + +# Define ARGs for versions (important for cache invalidation and clarity) +ARG ANSIBLE_VERSION="6.1.0" +ARG ANSIBLE_CORE_VERSION="2.13.13" +ARG POETRY_INSTALLER_VERSION="1.5.1" +ARG KUSTOMIZE_VER=5.3.0 + +# 1. Install common system dependencies first +RUN dnf install -y --allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ + dnf install -y --allowerasing \ + dnf-utils \ + curl \ + gnupg2 \ + make \ + gcc \ + parallel \ + sshpass \ + git \ + jq \ + ca-certificates \ + systemd \ + systemd-udev \ + dbus \ + && dnf clean all + +# 2. Install Python versions +# UBI9 default python3 is 3.9 is available via modules. +# Python 3.7 and 3.12 are not directly available in default repos for RHEL9/UBI9. +RUN dnf install -y --allowerasing \ + python3.9 \ + python3.9-devel \ + python3.9-pip \ + && dnf clean all + +# 4. Prepare python venv and install tools +RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ + # Create virtual environments in a common location + python3.9 -m venv /opt/.venv3.9 && \ + curl -sSL https://install.python-poetry.org | /opt/.venv3.9/bin/python - --version "${POETRY_INSTALLER_VERSION}" + +# 5. Install Ansible into the /opt/.venv3.9 virtual environment +RUN /opt/.venv3.9/bin/pip install --no-cache-dir \ + ansible=="${ANSIBLE_VERSION}" \ + ansible-core=="${ANSIBLE_CORE_VERSION}" \ + docker \ + pywinrm \ + requests + +# Add the /opt/.venv3.9/bin to PATH so 'ansible-playbook' etc. are found +ENV PATH="/opt/.venv3.9/bin:$PATH" + +# 6. Install Docker +# Add Docker's official repository for CentOS/RHEL +RUN dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ + dnf install -y --allowerasing \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + && dnf clean all + +# Add a 'docker' group (though in CI, often run as root) +RUN groupadd docker || true + +# 7. Install Podman (usually pre-installed or easily available) +RUN dnf install -y --allowerasing podman && \ + dnf clean all + +# Clean up any temporary files or caches created during the build to reduce image size +RUN dnf autoremove -y && \ + dnf clean all && \ + rm -rf /tmp/* /var/tmp/* + + +RUN systemctl mask systemd-logind.service \ + systemd-udevd.service \ + systemd-journald.service \ + systemd-resolved.service \ + networkd.service \ + getty@.service \ + graphical.target \ + multi-user.target \ + plymouth-quit-wait.service \ + plymouth-start.service \ + systemd-update-utmp-runlevel.service \ + systemd-update-utmp.service \ + systemd-vconsole-setup.service \ + console-getty.service || true + +ENTRYPOINT ["/usr/sbin/init"] \ No newline at end of file diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile new file mode 100644 index 0000000000..af793b8ad5 --- /dev/null +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:24.04 + +ENV container=docker \ + DEBIAN_FRONTEND=noninteractive + +RUN INSTALL_PKGS='findutils iproute2 python3 python3-apt sudo systemd' \ + && apt-get update && apt-get install $INSTALL_PKGS -y --no-install-recommends \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -print0 | xargs -0 rm -vf + +VOLUME [ "/sys/fs/cgroup" ] + +ENTRYPOINT [ "/lib/systemd/systemd" ] \ No newline at end of file