From 4b5c9d6ec1318aeb399147f11992b4d1e8f4da6c Mon Sep 17 00:00:00 2001 From: Szymon Bylica <142112966+sbylica-splunk@users.noreply.github.com> Date: Mon, 11 Aug 2025 14:16:23 +0200 Subject: [PATCH 01/91] Added docs for EKS deployment (#2785) * Added docs for EKS deployment * Changed some examples in Ansible documentation --- ansible/resources/env_file | 2 +- docs/gettingstarted/ansible-docker-podman.md | 8 +- docs/gettingstarted/ansible-docker-swarm.md | 2 +- docs/gettingstarted/eks.md | 106 +++++++++++++++++++ docs/resources/docker/sc4s_deployment.yaml | 85 +++++++++++++++ mkdocs.yml | 2 + 6 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 docs/gettingstarted/eks.md create mode 100644 docs/resources/docker/sc4s_deployment.yaml diff --git a/ansible/resources/env_file b/ansible/resources/env_file index 9f73a6d46c..3eb4275e4d 100644 --- a/ansible/resources/env_file +++ b/ansible/resources/env_file @@ -1,4 +1,4 @@ -SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://xxx.xxx.xxx.xxx:8088 +SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://xxx.xxx.xxx.xxx:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxxxxxxxxxxxx #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no \ No newline at end of file diff --git a/docs/gettingstarted/ansible-docker-podman.md b/docs/gettingstarted/ansible-docker-podman.md index 3f890ffa7a..1de2f6fd6b 100644 --- a/docs/gettingstarted/ansible-docker-podman.md +++ b/docs/gettingstarted/ansible-docker-podman.md @@ -23,16 +23,16 @@ docker exec -it ansible_sc4s /bin/bash * To authenticate with username and password: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --ask-pass path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --ask-pass ansible/playbooks/playbooks/podman.yml ``` * To authenticate using a key pair: ``` bash -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/docker.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/docker.yml or -ansible-playbook -i path/to/inventory.yaml -u --key-file path/to/playbooks/podman.yml +ansible-playbook -i ansible/inventory/inventory.yaml -u --key-file ansible/playbooks/playbooks/podman.yml ``` # Step 3: Validate your configuration diff --git a/docs/gettingstarted/ansible-docker-swarm.md b/docs/gettingstarted/ansible-docker-swarm.md index a34f4c1336..39417f3d96 100644 --- a/docs/gettingstarted/ansible-docker-swarm.md +++ b/docs/gettingstarted/ansible-docker-swarm.md @@ -51,7 +51,7 @@ ansible-playbook -i path/to/inventory_swarm.yaml -u --key-file Date: Thu, 21 Aug 2025 10:54:53 +0200 Subject: [PATCH 02/91] Fixed sonarqube issues (#2789) --- package/Dockerfile | 58 ++++++++-------- package/Dockerfile.enterprise | 65 ++++++++--------- package/Dockerfile.lite | 58 ++++++++-------- .../log_paths/2/lp_dest_alts_global/plugin.py | 7 +- .../conf.d/sources/source_syslog/plugin.py | 6 +- package/enterprise/etc/pylib/parser_cef.py | 5 +- .../enterprise/etc/pylib/parser_fix_dns.py | 5 +- package/enterprise/etc/pylib/parser_kvqf.py | 2 +- package/enterprise/etc/pylib/parser_leef.py | 69 +++++++++++-------- .../etc/pylib/parser_stealthbits.py | 10 +-- .../enterprise/etc/pylib/parser_vps_cache.py | 4 -- package/etc/pylib/parser_cef.py | 2 +- package/etc/pylib/parser_kvqf.py | 2 +- package/etc/pylib/parser_stealthbits.py | 1 - 14 files changed, 148 insertions(+), 146 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index c51d065327..c61bb433b2 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ diff --git a/package/Dockerfile.enterprise b/package/Dockerfile.enterprise index 93823cb1f3..40cb5e7849 100644 --- a/package/Dockerfile.enterprise +++ b/package/Dockerfile.enterprise @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ @@ -64,13 +64,14 @@ EXPOSE 6514/tcp #/dev/log a low priv user cannot read this and the container will fail in SC4S #and other uses the low user may be selected -HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5 +HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD ["/usr/sbin/syslog-ng-ctl", "healthcheck", "--timeout", "5"] COPY pyproject.toml / COPY poetry.lock / RUN python3 -m venv /var/lib/python-venv \ - && poetry export --format requirements.txt --without-hashes | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \ + && poetry export --format requirements.txt --without-hashes \ + | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \ && /var/lib/python-venv/bin/pip3 install --no-cache-dir --upgrade tornado==6.4.2 \ && apk del build-base python3-dev libffi-dev @@ -87,7 +88,7 @@ COPY package/sbin/source_ports_validator.py / ENV SC4S_CONTAINER_OPTS=--no-caps ARG VERSION=unknown -RUN echo $VERSION>/etc/syslog-ng/VERSION +RUN echo "$VERSION">/etc/syslog-ng/VERSION ENTRYPOINT ["/entrypoint.sh"] diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index f55a657882..61301205f9 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -21,35 +21,35 @@ FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} RUN apk add -U --upgrade --no-cache \ - bash \ - libxml2 \ - expat \ - binutils \ - musl \ - build-base \ - curl \ - grep \ - less \ - net-tools \ - netcat-openbsd \ - openssl \ - postgresql-libs \ - procps \ - py3-pip \ - python3 \ - python3-dev \ - libffi-dev \ - shadow \ - socat \ - tzdata \ - wget \ - cargo \ - ca-certificates \ - poetry \ - gdb \ - py3-poetry-plugin-export \ - py3-virtualenv \ - xz-libs \ + bash \ + binutils \ + build-base \ + ca-certificates \ + cargo \ + curl \ + expat \ + gdb \ + grep \ + less \ + libffi-dev \ + libxml2 \ + musl \ + net-tools \ + netcat-openbsd \ + openssl \ + poetry \ + postgresql-libs \ + procps \ + py3-pip \ + py3-poetry-plugin-export \ + py3-virtualenv \ + python3 \ + python3-dev \ + shadow \ + socat \ + tzdata \ + wget \ + xz-libs \ && groupadd --gid 1024 syslog \ && useradd -M -g 1024 -u 1024 syslog \ && usermod -L syslog \ diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py index 13bb14603d..76081f7f80 100755 --- a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py @@ -36,7 +36,7 @@ def normalize_env_variable_input(env_variable: str): modev = os.environ.get(f"SC4S_DEST_SPLUNK_HEC_{r}_MODE", "GLOBAL") if ( r == "DEFAULT" - and not os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") == "" + and os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") != "" ): if os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "yes").lower() in [ "true", @@ -68,9 +68,8 @@ def normalize_env_variable_input(env_variable: str): if r != "": modev = os.environ.get(f"SC4S_DEST_{t}_{r}_MODE", "GLOBAL") filter = os.environ.get(f"SC4S_DEST_{t}_{r}_FILTER", "") - if filter == "": - if t == "BSD": - filter = '"${MSG}" ne ""' + if filter == "" and t == "BSD": + filter = '"${MSG}" ne ""' if modev.upper() in ("GLOBAL", "SELECT"): global_dests[r] = { "destination": f"d_{t.lower()}_{r.lower()}", diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py index fdf790b0df..1d48a8a6b1 100755 --- a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py @@ -53,8 +53,6 @@ def normalize_env_variable_input(env_variable: str): if len(port_parts) == 2 or len(port_parts) == 3: vendor = port_parts[0].lower() product = port_parts[1].lower() - else: - pass outputText = tm.render( vendor=vendor, @@ -131,8 +129,8 @@ def normalize_env_variable_input(env_variable: str): "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", ), ebpf_no_sockets=int(os.getenv("SC4S_EBPF_NO_SOCKETS", 4)), - enable_parallelize=normalize_env_variable_input(f"SC4S_ENABLE_PARALLELIZE"), - parallelize_no_partitions=int(os.getenv(f"SC4S_PARALLELIZE_NO_PARTITION", 4)), + enable_parallelize=normalize_env_variable_input("SC4S_ENABLE_PARALLELIZE"), + parallelize_no_partitions=int(os.getenv("SC4S_PARALLELIZE_NO_PARTITION", 4)), set_source_sc4s=normalize_env_variable_input("SC4S_SET_SOURCE_AS_SC4S"), ) print(outputText) diff --git a/package/enterprise/etc/pylib/parser_cef.py b/package/enterprise/etc/pylib/parser_cef.py index 7d0673fd45..13e774960c 100644 --- a/package/enterprise/etc/pylib/parser_cef.py +++ b/package/enterprise/etc/pylib/parser_cef.py @@ -21,14 +21,13 @@ def parse(self, log_message): try: data = log_message.get_as_str(".metadata.cef.ext", "") - rpairs = re.findall(r"([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)", data) + rpairs = re.findall(r"([^=\s]+)=((?:\\=|[^=])+)(?:\s|$)", data) pairs = {} keys = [] for p in rpairs: pairs[p[0]] = p[1] keys.append(p[0]) - cleanpairs = {} for k in keys: if k.endswith("Label"): vk = k.rstrip("Label") @@ -50,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_fix_dns.py b/package/enterprise/etc/pylib/parser_fix_dns.py index de3bbd7d15..a35e30dbea 100644 --- a/package/enterprise/etc/pylib/parser_fix_dns.py +++ b/package/enterprise/etc/pylib/parser_fix_dns.py @@ -3,7 +3,6 @@ resolves IP to hostname value pair names are hard-coded """ -import re import socket try: @@ -25,7 +24,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + hostname, _, _ = socket.gethostbyaddr(ipaddr) if hostname == ipaddr: return False @@ -51,7 +50,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + fqdn, _, _ = socket.gethostbyaddr(ipaddr) if fqdn == ipaddr: return False diff --git a/package/enterprise/etc/pylib/parser_kvqf.py b/package/enterprise/etc/pylib/parser_kvqf.py index c5596c55f0..b174b7b6b0 100644 --- a/package/enterprise/etc/pylib/parser_kvqf.py +++ b/package/enterprise/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for matchNum, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/enterprise/etc/pylib/parser_leef.py b/package/enterprise/etc/pylib/parser_leef.py index 1279701314..5c5010cb2f 100644 --- a/package/enterprise/etc/pylib/parser_leef.py +++ b/package/enterprise/etc/pylib/parser_leef.py @@ -1,7 +1,5 @@ import re import binascii -import sys -import traceback try: import syslogng @@ -17,45 +15,64 @@ def init(self, options): self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))" self.logger = syslogng.Logger() return True + + def parse_message_from_pair(self, pair, log_message): + f, v = pair.split("=", 1) + if f == "devTime": + log_message[".leef." + f] = re.sub( + self.regex, "", v, 0, re.MULTILINE + ) + else: + log_message[".leef." + f] = v - def parse(self, log_message): + def parse_v1(self, log_message, event, structure, separator): + pairs = event.split(separator) + if len(pairs) < 4: + separator = "|" + pairs = structure[5:] + event = "\t".join(pairs) + log_message[".leef.event"] = event + return event, pairs, separator + + def parse_v2(self, event, structure, separator): + # V2 messages should always provide the sep but some fail do comply + # with the format spec if they don't assume tab + if len(structure) == 6 or not structure[5]: + pairs = event.split(separator) + else: + separator = structure[5] + if separator.startswith("0"): + separator = separator[1:] + pairs = event.split(separator) + return event, pairs, separator + def parse(self, log_message): try: msg = log_message.get_as_str("MESSAGE", "") # All LEEF message are | separated super structures structure = msg.split("|") - # Indexed fields for Splunk + # Indexed fields for Splunk log_message[".metadata.leef.version"] = structure[0][5:] log_message[".metadata.leef.vendor"] = structure[1] log_message[".metadata.leef.product"] = structure[2] log_message[".metadata.leef.product_version"] = structure[3] log_message[".metadata.leef.EventID"] = structure[4] + # We just want the event field event = structure[len(structure) - 1] log_message[".leef.event"] = event + + separator = "\t" + pairs = [] + # V1 will always use tab if structure[0][5:].startswith("1"): - separator = "\t" lv = "1" - pairs = event.split(separator) - if len(pairs) < 4: - separator = "|" - pairs = structure[5:] - event = "\t".join(pairs) - log_message[".leef.event"] = event + event, pairs, separator = self.parse_v1(log_message, event, structure, separator) else: lv = "2" - # V2 messages should always provide the sep but some fail do comply - # with the format spec if they don't assume tab - if len(structure) == 6 or not structure[5]: - separator = "\t" - pairs = event.split(separator) - else: - separator = structure[5] - if separator.startswith("0"): - separator = separator[1:] - pairs = event.split(separator) + event, pairs, separator = self.parse_v2(event, structure, separator) if separator.startswith("x"): hex_sep = f"0{separator.lower()}" @@ -70,15 +87,9 @@ def parse(self, log_message): log_message["fields.sc4s_product"] = structure[2] for p in pairs: - f, v = p.split("=", 1) - if f == "devTime": - log_message[".leef." + f] = re.sub( - self.regex, "", v, 0, re.MULTILINE - ) - else: - log_message[".leef." + f] = v + self.parse_message_from_pair(p, log_message) except Exception as e: log_message[".metadata.leef.exception"] = str(e) # return True, other way message is dropped - return True \ No newline at end of file + return True diff --git a/package/enterprise/etc/pylib/parser_stealthbits.py b/package/enterprise/etc/pylib/parser_stealthbits.py index bdf0369816..89445d5119 100644 --- a/package/enterprise/etc/pylib/parser_stealthbits.py +++ b/package/enterprise/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: @@ -10,6 +9,7 @@ class LogParser: regex = r"^(.*[\.\!\?])?(.*:.*)" +alert_text_key = ".values.AlertText" class alerttext_kv(LogParser): @@ -17,13 +17,13 @@ def init(self, options): return True def parse(self, log_message): - match = re.search(regex, log_message.get_as_str(".values.AlertText", "")) + match = re.search(regex, log_message.get_as_str(alert_text_key, "")) if match: - log_message[".values.AlertText"] = match.groups()[0] + log_message[alert_text_key] = match.groups()[0] text = match.groups()[1] else: - text = log_message.get_as_str(".values.AlertText", "") - log_message[".values.AlertText"] = "" + text = log_message.get_as_str(alert_text_key, "") + log_message[alert_text_key] = "" pairs = text.split("; ") diff --git a/package/enterprise/etc/pylib/parser_vps_cache.py b/package/enterprise/etc/pylib/parser_vps_cache.py index 4c8cf21250..d545620158 100644 --- a/package/enterprise/etc/pylib/parser_vps_cache.py +++ b/package/enterprise/etc/pylib/parser_vps_cache.py @@ -95,7 +95,3 @@ def send(self, log_message): def flush(self): self.db.commit() return True - - -if __name__ == "__main__": - pass diff --git a/package/etc/pylib/parser_cef.py b/package/etc/pylib/parser_cef.py index 28fd68498e..32dede7e12 100644 --- a/package/etc/pylib/parser_cef.py +++ b/package/etc/pylib/parser_cef.py @@ -49,5 +49,5 @@ def parse(self, log_message): lines = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logger.debug("".join("!! " + line for line in lines)) return False - self.logger.debug("kvqf_parse.parse complete") + return True diff --git a/package/etc/pylib/parser_kvqf.py b/package/etc/pylib/parser_kvqf.py index 99206af8d1..b7d3253681 100644 --- a/package/etc/pylib/parser_kvqf.py +++ b/package/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for match_num, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v diff --git a/package/etc/pylib/parser_stealthbits.py b/package/etc/pylib/parser_stealthbits.py index 07a3c87f10..99800d479f 100644 --- a/package/etc/pylib/parser_stealthbits.py +++ b/package/etc/pylib/parser_stealthbits.py @@ -1,7 +1,6 @@ import re try: - import syslogng from syslogng import LogParser except Exception: From b9539f31476321e22ec95a3e6a9a9b680abb56c4 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 11:27:54 +0530 Subject: [PATCH 03/91] chore: running with initial ansible changes --- .github/workflows/ci-enterprise.yaml | 50 ++++++++++++---------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 10afd7a00e..81952be1c7 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -202,41 +202,33 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + RUN pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir + - name: Installing system dependencies for Ansible + run: | + RUN apt update -y \ + && apt install --no-install-recommends -y build-essential parallel \ + && apt install --no-install-recommends -y sshpass \ + && apt clean + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + - name: Run Ansible Playbook (Example) + run: | + ansible-playbook -i ansible/inventory.ini ansible/playbooks/your_main_playbook.yml - name: Run tests run: | pip3 install poetry From 2bb798f4b5a820703e501dcaca66446aa8645323 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 11:35:48 +0530 Subject: [PATCH 04/91] chore: update ci-main instead of ci-enterprise for testing --- .github/workflows/ci-enterprise.yaml | 50 +++++++++++++++------------ .github/workflows/ci-main.yaml | 51 ++++++++++++---------------- 2 files changed, 50 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 81952be1c7..10afd7a00e 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -202,33 +202,41 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz + sc4s: + image: ${{ needs.meta.outputs.container_base }} + ports: + - 514:514 + - 601:601 + - 5614:5514 + - 5601:5601 + - 6000:6000 + - 6002:6002 + - 9000:9000 + env: + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" + TEST_SC4S_ACTIVATE_EXAMPLES: "yes" + SC4S_DEBUG_CONTAINER: "yes" + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" + SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Install Ansible and other dependencies as python package - run: | - RUN pip install ansible~=6.1.0 --no-cache-dir \ - && pip install pywinrm>=0.4.2 --no-cache-dir \ - && pip install ansible-lint>=6.0.0 --no-cache-dir - - name: Installing system dependencies for Ansible - run: | - RUN apt update -y \ - && apt install --no-install-recommends -y build-essential parallel \ - && apt install --no-install-recommends -y sshpass \ - && apt clean - - name: Configure Ansible Environment Variables - env: - ANSIBLE_CONFIG: ansible.cfg - ANSIBLE_HOST_KEY_CHECKING: False - run: | - echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" - echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - - name: Run Ansible Playbook (Example) - run: | - ansible-playbook -i ansible/inventory.ini ansible/playbooks/your_main_playbook.yml - name: Run tests run: | pip3 install poetry diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index d0d1c137b5..ed4fdee301 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -200,42 +200,33 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + RUN pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir + - name: Installing system dependencies for Ansible + run: | + RUN apt update -y \ + && apt install --no-install-recommends -y build-essential parallel \ + && apt install --no-install-recommends -y sshpass \ + && apt clean + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + - name: Run Ansible Playbook (Example) + run: | + ansible-playbook -i ansible/inventory.ini ansible/playbooks/your_main_playbook.yml - name: Run tests run: | pip3 install poetry From c665d59e46564527206143cf2890bf646190b811 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 11:41:52 +0530 Subject: [PATCH 05/91] chore: remove RUN command --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index ed4fdee301..e35fee0a14 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -208,12 +208,12 @@ jobs: persist-credentials: false - name: Install Ansible and other dependencies as python package run: | - RUN pip install ansible~=6.1.0 --no-cache-dir \ + pip install ansible~=6.1.0 --no-cache-dir \ && pip install pywinrm>=0.4.2 --no-cache-dir \ && pip install ansible-lint>=6.0.0 --no-cache-dir - name: Installing system dependencies for Ansible run: | - RUN apt update -y \ + apt update -y \ && apt install --no-install-recommends -y build-essential parallel \ && apt install --no-install-recommends -y sshpass \ && apt clean From 28d397341019282a6c2a6cb79cd261ca0a0ffd90 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 12:46:50 +0530 Subject: [PATCH 06/91] chore: update base debian image for testing --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e35fee0a14..8430afef06 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -185,7 +185,7 @@ jobs: # runs all of the steps inside the specified container rather than on the VM host. # Because of this the network configuration changes from host based network to a container network. container: - image: python:3.9-buster + image: python:3.9-bookworm services: splunk: From 0b447b6f4b89e81e737b4a7b828bbbeecb141794 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 13:58:04 +0530 Subject: [PATCH 07/91] chore: update env_file and inventory file --- .github/workflows/ci-main.yaml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 8430afef06..d4448d64ab 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -224,9 +224,27 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - - name: Run Ansible Playbook (Example) + - name: Update inventory file and env_file run: | - ansible-playbook -i ansible/inventory.ini ansible/playbooks/your_main_playbook.yml + cat << EOF > ansible/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + EOF + echo "Updated ansible/inventory/inventory.yaml" + echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088" >> ansible/resources/env_file + echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" >> ansible/resources/env_file + echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no" >> ansible/resources/env_file + echo "Updated ansible/resources/env_file" + + - name: Run Ansible Playbook + run: | + ansible-playbook -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml - name: Run tests run: | pip3 install poetry From ff2015b2a99cbbe7344c983e05166c42fde953b1 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 15:39:21 +0530 Subject: [PATCH 08/91] chore: run ansible on docker host --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index d4448d64ab..1a87a07a75 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -235,6 +235,7 @@ jobs: hosts: node_1: ansible_host: 127.0.0.1 + ansible_connection: docker EOF echo "Updated ansible/inventory/inventory.yaml" echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088" >> ansible/resources/env_file From 64d24d5378021d9ef7912fcd29658ae741b7f12d Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:00:49 +0530 Subject: [PATCH 09/91] chore: test local --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 1a87a07a75..e0abfa2632 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -235,7 +235,7 @@ jobs: hosts: node_1: ansible_host: 127.0.0.1 - ansible_connection: docker + ansible_connection: local EOF echo "Updated ansible/inventory/inventory.yaml" echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088" >> ansible/resources/env_file @@ -245,7 +245,7 @@ jobs: - name: Run Ansible Playbook run: | - ansible-playbook -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml - name: Run tests run: | pip3 install poetry From faaa429840eb3c7636edaed23506322d6f28aa3b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:11:54 +0530 Subject: [PATCH 10/91] chore: skip docker install via python --- ansible/roles/install_docker/tasks/install_docker_debian.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..80792a3fea 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -28,10 +28,9 @@ name: docker-ce state: latest -- name: Install Docker Module for Python +- name: Install Modules for Python pip: name: - - docker - jsondiff - name: Start Docker From 8d4b6c271d7b03310be53d215b782224f0c2bd29 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:28:14 +0530 Subject: [PATCH 11/91] chore: test --- .github/workflows/ci-main.yaml | 7 ++++++- .../roles/install_docker/tasks/install_docker_debian.yml | 5 ----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e0abfa2632..f72f33b6ed 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -226,7 +226,7 @@ jobs: echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - name: Update inventory file and env_file run: | - cat << EOF > ansible/inventory.yaml + cat << EOF > ansible/inventory/inventory.yaml --- all: hosts: @@ -243,6 +243,11 @@ jobs: echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no" >> ansible/resources/env_file echo "Updated ansible/resources/env_file" + - name: Debug + run: | + cat /etc/os-release + cat ansible/inventory/inventory.yaml + cat ansible/inventory/inventory.yaml - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 80792a3fea..cd8ae97870 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -28,11 +28,6 @@ name: docker-ce state: latest -- name: Install Modules for Python - pip: - name: - - jsondiff - - name: Start Docker ansible.builtin.service: name: "docker" From e2233197036f8844531080cc4f2ffaa66a780868 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:33:46 +0530 Subject: [PATCH 12/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index f72f33b6ed..5b2c50d194 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -250,7 +250,7 @@ jobs: cat ansible/inventory/inventory.yaml - name: Run Ansible Playbook run: | - ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + ansible-playbook --connection=local ansible/playbooks/docker.yml - name: Run tests run: | pip3 install poetry From 5877c4dfab5b98721614f12520e9ccd133eecca5 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:46:41 +0530 Subject: [PATCH 13/91] chore: test --- .github/workflows/ci-main.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5b2c50d194..8440126ddf 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -224,7 +224,7 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - - name: Update inventory file and env_file + - name: Update inventory file run: | cat << EOF > ansible/inventory/inventory.yaml --- @@ -236,7 +236,8 @@ jobs: node_1: ansible_host: 127.0.0.1 ansible_connection: local - EOF + - name: Update env_file + run: | echo "Updated ansible/inventory/inventory.yaml" echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088" >> ansible/resources/env_file echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" >> ansible/resources/env_file @@ -247,10 +248,10 @@ jobs: run: | cat /etc/os-release cat ansible/inventory/inventory.yaml - cat ansible/inventory/inventory.yaml + cat ansible/resources/env_file - name: Run Ansible Playbook run: | - ansible-playbook --connection=local ansible/playbooks/docker.yml + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml - name: Run tests run: | pip3 install poetry From 746f3016749030ed0b71f51aaa8021ecd527cd7e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 17:57:07 +0530 Subject: [PATCH 14/91] chore: test --- .github/workflows/ci-main.yaml | 10 +++++----- .../install_docker/tasks/install_docker_debian.yml | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 8440126ddf..451edfb096 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -238,11 +238,11 @@ jobs: ansible_connection: local - name: Update env_file run: | - echo "Updated ansible/inventory/inventory.yaml" - echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088" >> ansible/resources/env_file - echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" >> ansible/resources/env_file - echo "SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no" >> ansible/resources/env_file - echo "Updated ansible/resources/env_file" + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no - name: Debug run: | diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index cd8ae97870..e9282f03a9 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -28,6 +28,12 @@ name: docker-ce state: latest +- name: Install Docker Module for Python + apt: + name: + - python-docker + - python-jsondiff + - name: Start Docker ansible.builtin.service: name: "docker" From d464d5d53515867557924708c904f0e3adcd0ff0 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 18:03:50 +0530 Subject: [PATCH 15/91] chore: test running directly on ubuntu GH runner --- .github/workflows/ci-main.yaml | 4 ---- .../roles/install_docker/tasks/install_docker_debian.yml | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 451edfb096..79f7bf3e42 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -182,10 +182,6 @@ jobs: needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-bookworm services: splunk: diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index e9282f03a9..21a517907d 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -29,10 +29,10 @@ state: latest - name: Install Docker Module for Python - apt: + pip: name: - - python-docker - - python-jsondiff + - docker + - jsondiff - name: Start Docker ansible.builtin.service: From 62ea7c33be807e4392365293e55e0234621174d2 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 18:09:27 +0530 Subject: [PATCH 16/91] chore: test --- .github/workflows/ci-main.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 79f7bf3e42..5fc64866fa 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -207,12 +207,12 @@ jobs: pip install ansible~=6.1.0 --no-cache-dir \ && pip install pywinrm>=0.4.2 --no-cache-dir \ && pip install ansible-lint>=6.0.0 --no-cache-dir - - name: Installing system dependencies for Ansible - run: | - apt update -y \ - && apt install --no-install-recommends -y build-essential parallel \ - && apt install --no-install-recommends -y sshpass \ - && apt clean + # - name: Installing system dependencies for Ansible + # run: | + # apt update -y \ + # && apt install --no-install-recommends -y build-essential parallel \ + # && apt install --no-install-recommends -y sshpass \ + # && apt clean - name: Configure Ansible Environment Variables env: ANSIBLE_CONFIG: ansible.cfg From eb0ada09f337256416c41157d467f1bca19fd971 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 10 Sep 2025 18:29:18 +0530 Subject: [PATCH 17/91] chore: test --- .github/workflows/ci-main.yaml | 2 ++ ansible/tasks/docker/deploy_app.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5fc64866fa..ac2b88171a 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -242,6 +242,8 @@ jobs: - name: Debug run: | + whoami + uname -a cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 50462fac88..199eda1503 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -14,16 +14,16 @@ copy: src: "/opt/ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" + owner: "root" + group: "root" mode: u=rw,g=rw - name: Copying env_file file on the server copy: src: "/opt/ansible/resources/env_file" dest: "/opt/sc4s/env_file" - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" + owner: "root" + group: "root" mode: u=rw,g=rw - name: Create a volume From fb4c36bb36f1263d7ee7e72436b185ae57f6456e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 10:58:07 +0530 Subject: [PATCH 18/91] chore: test --- ansible/tasks/docker/deploy_app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 199eda1503..58c10ff53e 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/docker_sc4s.service" + src: "ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "root" group: "root" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "root" group: "root" From a17753e440229b77e67c64ddc086b4ef744991cd Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 11:13:29 +0530 Subject: [PATCH 19/91] chore: test --- ansible/tasks/docker/deploy_app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 58c10ff53e..6e090fa1c9 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "ansible/resources/docker_sc4s.service" + src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "root" group: "root" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "root" group: "root" From 4bd3fc5d5ff1085d59bb3ea78230a3bdc16437e7 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 12:09:38 +0530 Subject: [PATCH 20/91] chore: try localhost --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index ac2b88171a..0ad2d8e13d 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -236,7 +236,7 @@ jobs: run: | echo "Updating ansible/inventory/inventory.yaml" cat << EOF > ansible/resources/env_file - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no From 463eb8e1564bff5af7f8c862c3455368126842db Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 12:26:13 +0530 Subject: [PATCH 21/91] chore: export artifact --- .github/workflows/ci-main.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 0ad2d8e13d..c1c0eafd03 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -263,6 +263,12 @@ jobs: --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-ipv4-name-cache: runs-on: ubuntu-latest From 9da54ae7e389e4333d4d3ad3c8389ef5c615d288 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 14:48:20 +0530 Subject: [PATCH 22/91] chore: test again --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c1c0eafd03..97eca9e2b6 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -258,8 +258,8 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=localhost \ + --sc4s_host=localhost \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' From c5effa05f6f68d834fab57c5cde305ef7f43311c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 15:18:21 +0530 Subject: [PATCH 23/91] chore: debug --- .github/workflows/ci-main.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 97eca9e2b6..8e018ddc9a 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -250,6 +250,13 @@ jobs: - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + - name: Debug 2 + run: | + whoami + uname -a + docker ps + systemctl status dokcer_sc4s + docker images - name: Run tests run: | pip3 install poetry @@ -258,8 +265,8 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=localhost \ - --sc4s_host=localhost \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' From 7c7c6a7224e6778eaf22616b1e99ee00d248f67a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 15:26:44 +0530 Subject: [PATCH 24/91] chore: debug again --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 8e018ddc9a..cdd8756543 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -255,7 +255,7 @@ jobs: whoami uname -a docker ps - systemctl status dokcer_sc4s + systemctl status sc4s docker images - name: Run tests run: | From c9c047aa66c5a377e343cd480f7cd18a8e439709 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 15:57:10 +0530 Subject: [PATCH 25/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index cdd8756543..2274365b17 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -247,6 +247,7 @@ jobs: cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file + docker ps - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml From 1d745306e9807a7e6fad1d8813964a3f361488c7 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 16:09:18 +0530 Subject: [PATCH 26/91] chore: ignore docker installation role if docker already exists --- ansible/playbooks/docker.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index 655405449b..7633d66017 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -4,7 +4,14 @@ vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: + - name: Check if Docker is installed + community.docker.docker_info: + register: docker_status + ignore_errors: true + - name: Docker installation role include_role: name: install_docker - - include_tasks: ../tasks/docker/deploy_app.yml + when: not docker_status.exists # Include role if Docker is NOT found + + - include_tasks: ../tasks/docker/deploy_app.ym From b0333b186501db103c8cf3073a71262c9ad8f1ae Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 17:36:28 +0530 Subject: [PATCH 27/91] chore: tests --- ansible/playbooks/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index 7633d66017..bdf6986076 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -4,14 +4,14 @@ vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: - - name: Check if Docker is installed - community.docker.docker_info: - register: docker_status + - name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check ignore_errors: true - name: Docker installation role include_role: name: install_docker - when: not docker_status.exists # Include role if Docker is NOT found + when: docker_command_check.rc != 0 - - include_tasks: ../tasks/docker/deploy_app.ym + - include_tasks: ../tasks/docker/deploy_app.yml From d0c9e77bc8d05acc2d5fec1e93a5c5a6e3e36ab8 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 11 Sep 2025 17:45:15 +0530 Subject: [PATCH 28/91] chore: tests --- ansible/playbooks/docker.yml | 7 ------- .../roles/install_docker/tasks/install_docker_debian.yml | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index bdf6986076..655405449b 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -4,14 +4,7 @@ vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: - - name: Check if 'docker' command exists - ansible.builtin.command: which docker - register: docker_command_check - ignore_errors: true - - name: Docker installation role include_role: name: install_docker - when: docker_command_check.rc != 0 - - include_tasks: ../tasks/docker/deploy_app.yml diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..d3d434a774 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -23,10 +23,16 @@ repo: deb https://download.docker.com/linux/ubuntu focal stable state: present +- name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true + - name: Install docker-ce apt: name: docker-ce state: latest + when: docker_command_check.rc != 0 - name: Install Docker Module for Python pip: From c1a2ddb11d2fba758bdcdceaf72a0d0f1dd27d3f Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 10:59:27 +0530 Subject: [PATCH 29/91] chore: test --- .github/workflows/ci-main.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 2274365b17..2973626ea9 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,6 +178,17 @@ jobs: scanners: "vuln" test-container: + strategy: + fail-fast: false + matrix: + # Use official base images for the OSes + container_os_image: + - ubuntu:24.04 + - centos:7 + - redhat/ubi8 + container: + image: ${{ matrix.container_os_image }} + options: --privileged -v /var/run/docker.sock:/var/run/docker.sock runs-on: ubuntu-latest needs: - meta From 20de2c05486b9d440b3d766c174010dfc5a3f79a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 11:09:31 +0530 Subject: [PATCH 30/91] chore: test --- .github/workflows/ci-main.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 2973626ea9..ce04c2f593 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -208,6 +208,9 @@ jobs: SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz steps: + - uses: actions/setup-python@v5 + with: + python-version: 3.10 - name: Checkout uses: actions/checkout@v4 with: From d82e4022986a0de3d746d8338ce0594768cfed09 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 11:15:40 +0530 Subject: [PATCH 31/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index ce04c2f593..75f112bf6c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -210,7 +210,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: 3.9.23 - name: Checkout uses: actions/checkout@v4 with: From 32dc9816f4bc6eef344eed8ec9cb7fc7635773d1 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 11:22:18 +0530 Subject: [PATCH 32/91] chore: debug --- .github/workflows/ci-main.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 75f112bf6c..4a76325953 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -208,6 +208,11 @@ jobs: SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz steps: + - name: Debug -2 + run: | + cat /etc/os-release + uname -a + which python - uses: actions/setup-python@v5 with: python-version: 3.9.23 @@ -216,9 +221,14 @@ jobs: with: submodules: false persist-credentials: false + - name: Debug -1 + run: | + cat /etc/os-release + uname -a + which python - name: Install Ansible and other dependencies as python package run: | - pip install ansible~=6.1.0 --no-cache-dir \ + python -m pip install ansible~=6.1.0 --no-cache-dir \ && pip install pywinrm>=0.4.2 --no-cache-dir \ && pip install ansible-lint>=6.0.0 --no-cache-dir # - name: Installing system dependencies for Ansible From 1fb9adedcda14c35652f9202b929e4720de7bd71 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 11:28:17 +0530 Subject: [PATCH 33/91] chore: test --- .github/workflows/ci-main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 4a76325953..c658ba264a 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -212,7 +212,6 @@ jobs: run: | cat /etc/os-release uname -a - which python - uses: actions/setup-python@v5 with: python-version: 3.9.23 From 2f727811d46318e3e52632bbdc9ca7f2ab931793 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 11:37:49 +0530 Subject: [PATCH 34/91] chore: fix pip --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c658ba264a..1841f4904f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -228,8 +228,8 @@ jobs: - name: Install Ansible and other dependencies as python package run: | python -m pip install ansible~=6.1.0 --no-cache-dir \ - && pip install pywinrm>=0.4.2 --no-cache-dir \ - && pip install ansible-lint>=6.0.0 --no-cache-dir + && python -m pip install pywinrm>=0.4.2 --no-cache-dir \ + && python -m pip install ansible-lint>=6.0.0 --no-cache-dir # - name: Installing system dependencies for Ansible # run: | # apt update -y \ From 10cfc4caee726b23aecfdbca717f9de004474448 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 12:05:55 +0530 Subject: [PATCH 35/91] chore: test --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 1841f4904f..5c1802b4ca 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -185,7 +185,7 @@ jobs: container_os_image: - ubuntu:24.04 - centos:7 - - redhat/ubi8 + - redhat/ubi8/python-39 container: image: ${{ matrix.container_os_image }} options: --privileged -v /var/run/docker.sock:/var/run/docker.sock @@ -270,7 +270,7 @@ jobs: cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file - docker ps + # docker ps - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml From 22cd7f0a6fe910176a34ccd78d21f8f5e8cf5772 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 12:28:53 +0530 Subject: [PATCH 36/91] chore: test --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5c1802b4ca..14dea1e9eb 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -184,8 +184,8 @@ jobs: # Use official base images for the OSes container_os_image: - ubuntu:24.04 - - centos:7 - - redhat/ubi8/python-39 + - centos:8 + - redhat/ubi9 container: image: ${{ matrix.container_os_image }} options: --privileged -v /var/run/docker.sock:/var/run/docker.sock From 7ad18ee0e793fef71a9c6b5512472eb6c81d73d3 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 12:36:24 +0530 Subject: [PATCH 37/91] chore: test --- .github/workflows/ci-main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 14dea1e9eb..51bfbde649 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -185,7 +185,7 @@ jobs: container_os_image: - ubuntu:24.04 - centos:8 - - redhat/ubi9 + - redhat/ubi10 container: image: ${{ matrix.container_os_image }} options: --privileged -v /var/run/docker.sock:/var/run/docker.sock @@ -224,7 +224,6 @@ jobs: run: | cat /etc/os-release uname -a - which python - name: Install Ansible and other dependencies as python package run: | python -m pip install ansible~=6.1.0 --no-cache-dir \ From 861bf7b1528f1672f2b2724a50e3b38f7d2f7ed7 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 13:55:40 +0530 Subject: [PATCH 38/91] chore: test --- .github/workflows/ci-main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 51bfbde649..c3137a4a0b 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -188,7 +188,6 @@ jobs: - redhat/ubi10 container: image: ${{ matrix.container_os_image }} - options: --privileged -v /var/run/docker.sock:/var/run/docker.sock runs-on: ubuntu-latest needs: - meta From 025b5a918097b648105b3b2a957b90c65cbae8b4 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 15:56:39 +0530 Subject: [PATCH 39/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c3137a4a0b..51bfbde649 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -188,6 +188,7 @@ jobs: - redhat/ubi10 container: image: ${{ matrix.container_os_image }} + options: --privileged -v /var/run/docker.sock:/var/run/docker.sock runs-on: ubuntu-latest needs: - meta From 33129f2f436ad21dd9af079788835b3d7d24d6d1 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 16:04:40 +0530 Subject: [PATCH 40/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 51bfbde649..59632de892 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -184,7 +184,7 @@ jobs: # Use official base images for the OSes container_os_image: - ubuntu:24.04 - - centos:8 + - dokken/centos-stream-9 - redhat/ubi10 container: image: ${{ matrix.container_os_image }} From 15248425a40a4b633fca94436c9080ce5d99b816 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 16:13:54 +0530 Subject: [PATCH 41/91] chore: test --- .github/workflows/ci-main.yaml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 59632de892..851c1f2c99 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -212,9 +212,23 @@ jobs: run: | cat /etc/os-release uname -a - - uses: actions/setup-python@v5 - with: - python-version: 3.9.23 + - name: Install system dependencies + run: | + if [ -f /etc/debian_version ]; then + echo "Detected Debian/Ubuntu-based OS: ${{ matrix.container_os_image }}" + apt update -y + apt install --no-install-recommends -y python3 python3-pip build-essential parallel sshpass curl gnupg # Added gnupg + apt clean + elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then + echo "Detected RHEL/CentOS-based OS: ${{ matrix.container_os_image }}" + yum update -y || dnf update -y + # Added gnupg2 for RHEL/CentOS systems + yum install -y python3 python3-pip make gcc parallel sshpass curl gnupg2 || dnf install -y python3 python3-pip make gcc parallel sshpass curl gnupg2 + yum clean all || dnf clean all + else + echo "Unsupported OS detected inside container for ${{ matrix.container_os_image }}. Please add installation commands." + exit 1 + fi - name: Checkout uses: actions/checkout@v4 with: From 4bdfeb8c2a9905d58c9829d42c01fdf92a61ebe4 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 16:22:19 +0530 Subject: [PATCH 42/91] chore: test --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 851c1f2c99..1d66b89578 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -217,13 +217,13 @@ jobs: if [ -f /etc/debian_version ]; then echo "Detected Debian/Ubuntu-based OS: ${{ matrix.container_os_image }}" apt update -y - apt install --no-install-recommends -y python3 python3-pip build-essential parallel sshpass curl gnupg # Added gnupg + apt install --no-install-recommends -y python3 python3-pip build-essential curl gnupg # Added gnupg apt clean elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then echo "Detected RHEL/CentOS-based OS: ${{ matrix.container_os_image }}" yum update -y || dnf update -y # Added gnupg2 for RHEL/CentOS systems - yum install -y python3 python3-pip make gcc parallel sshpass curl gnupg2 || dnf install -y python3 python3-pip make gcc parallel sshpass curl gnupg2 + yum install -y python3 python3-pip make gcc curl gnupg2 || dnf install -y python3 python3-pip make gcc curl gnupg2 yum clean all || dnf clean all else echo "Unsupported OS detected inside container for ${{ matrix.container_os_image }}. Please add installation commands." From e1c456341d13603b9d088329d21d195a14ff690d Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 17:16:55 +0530 Subject: [PATCH 43/91] chore: test --- .github/workflows/ci-main.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 1d66b89578..9b32896cee 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -217,18 +217,21 @@ jobs: if [ -f /etc/debian_version ]; then echo "Detected Debian/Ubuntu-based OS: ${{ matrix.container_os_image }}" apt update -y - apt install --no-install-recommends -y python3 python3-pip build-essential curl gnupg # Added gnupg + apt install --no-install-recommends -y python3-pip build-essential curl gnupg # Added gnupg apt clean elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then echo "Detected RHEL/CentOS-based OS: ${{ matrix.container_os_image }}" yum update -y || dnf update -y # Added gnupg2 for RHEL/CentOS systems - yum install -y python3 python3-pip make gcc curl gnupg2 || dnf install -y python3 python3-pip make gcc curl gnupg2 + yum install -y python3-pip make gcc curl gnupg2 || dnf install -y python3-pip make gcc curl gnupg2 yum clean all || dnf clean all else echo "Unsupported OS detected inside container for ${{ matrix.container_os_image }}. Please add installation commands." exit 1 fi + - uses: actions/setup-python@v5 + with: + python-version: 3.9.23 - name: Checkout uses: actions/checkout@v4 with: From b9ed016e304af6de939dd396fdd3bae666bff0f5 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 17:30:05 +0530 Subject: [PATCH 44/91] chore: test --- .github/workflows/ci-main.yaml | 5 +++++ ansible/roles/install_docker/tasks/install_docker_rhel.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 9b32896cee..e2bf11fd7f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -232,6 +232,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.9.23 + - name: Create and activate virtual environment + run: | + python -m venv .venv # Creates a virtual environment named '.venv' + echo "VIRTUAL_ENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV # Store path for later use + echo "PATH=$(pwd)/.venv/bin:$PATH" >> $GITHUB_ENV # Add venv bin to PATH for subsequent steps - name: Checkout uses: actions/checkout@v4 with: diff --git a/ansible/roles/install_docker/tasks/install_docker_rhel.yml b/ansible/roles/install_docker/tasks/install_docker_rhel.yml index 48a9f6e9bb..11e4808cf8 100644 --- a/ansible/roles/install_docker/tasks/install_docker_rhel.yml +++ b/ansible/roles/install_docker/tasks/install_docker_rhel.yml @@ -11,7 +11,7 @@ state: present - name: Add Docker Repository (RedHat, centOS) - shell: | + _uses_shell: | dnf install -y yum-utils yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo From 7802a07feea09b832c0f5c4b18a099742a8caf6e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 17:37:57 +0530 Subject: [PATCH 45/91] chore: test --- ansible/roles/install_docker/tasks/install_docker_rhel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/install_docker/tasks/install_docker_rhel.yml b/ansible/roles/install_docker/tasks/install_docker_rhel.yml index 11e4808cf8..48a9f6e9bb 100644 --- a/ansible/roles/install_docker/tasks/install_docker_rhel.yml +++ b/ansible/roles/install_docker/tasks/install_docker_rhel.yml @@ -11,7 +11,7 @@ state: present - name: Add Docker Repository (RedHat, centOS) - _uses_shell: | + shell: | dnf install -y yum-utils yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo From 1df0e7d9815e3890009b6b8b049b839b9e88b20a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 12 Sep 2025 18:05:45 +0530 Subject: [PATCH 46/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- .../roles/install_docker/tasks/install_docker_rhel.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e2bf11fd7f..45dde98abf 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -217,7 +217,7 @@ jobs: if [ -f /etc/debian_version ]; then echo "Detected Debian/Ubuntu-based OS: ${{ matrix.container_os_image }}" apt update -y - apt install --no-install-recommends -y python3-pip build-essential curl gnupg # Added gnupg + apt install --no-install-recommends -y python3-pip python3-docker python3-jsondiff build-essential curl gnupg # Added gnupg apt clean elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then echo "Detected RHEL/CentOS-based OS: ${{ matrix.container_os_image }}" diff --git a/ansible/roles/install_docker/tasks/install_docker_rhel.yml b/ansible/roles/install_docker/tasks/install_docker_rhel.yml index 48a9f6e9bb..d3938202e0 100644 --- a/ansible/roles/install_docker/tasks/install_docker_rhel.yml +++ b/ansible/roles/install_docker/tasks/install_docker_rhel.yml @@ -28,11 +28,11 @@ state: latest update_cache: true -- name: Install Docker Module for Python - pip: - name: - - docker - - jsondiff +# - name: Install Docker Module for Python +# pip: +# name: +# - docker +# - jsondiff - name: Start Docker ansible.builtin.service: From 1d83d5eab2ce4cbf689f30860a8ebc874c12b641 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 17 Sep 2025 16:12:31 +0530 Subject: [PATCH 47/91] chore: test building env images --- .github/workflows/ci-env-images.yaml | 76 +++++++++++ .../docker/centos_7/Dockerfile | 0 .../docker/redhat_9/Dockerfile | 98 ++++++++++++++ .../docker/ubuntu_24/Dockerfile | 125 ++++++++++++++++++ 4 files changed, 299 insertions(+) create mode 100644 .github/workflows/ci-env-images.yaml create mode 100644 tests/container_images/docker/centos_7/Dockerfile create mode 100644 tests/container_images/docker/redhat_9/Dockerfile create mode 100644 tests/container_images/docker/ubuntu_24/Dockerfile diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml new file mode 100644 index 0000000000..9a16721252 --- /dev/null +++ b/.github/workflows/ci-env-images.yaml @@ -0,0 +1,76 @@ +name: Build and Publish CI Environment Images + +on: + pull_request: + branches: + - develop + paths: + - 'tests/container_images/docker/ubuntu_24/Dockerfile' + - 'tests/container_images/docker/redhat_9/Dockerfile' + + +jobs: + build-and-push-ci-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Required to push to GHCR + + strategy: + fail-fast: false # Set to false to build all images even if one fails + matrix: + os_variant: [ubuntu, rhel, centos] + include: + - os_variant: ubuntu + dockerfile_path: tests/container_images/docker/ubuntu_24/Dockerfile + image_name_suffix: ci-env-ubuntu + # Build args specific to Ubuntu + ansible_version: "6.1.0" + ansible_core_version: "2.13.13" + poetry_installer_version: "1.5.1" + - os_variant: rhel + dockerfile_path: tests/container_images/docker/redhat_9/Dockerfile + image_name_suffix: ci-env-rhel + # Build args specific to RHEL + ansible_version: "6.1.0" + ansible_core_version: "2.13.13" + poetry_installer_version: "1.5.1" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }} + tags: | + type=ref,event=pr,suffix=-pr{{pr.number}} # e.g., ci-env-ubuntu:pr-123 + type=sha,suffix=-{{sha}} # e.g., ci-env-ubuntu:abcdef123456 + + - name: Build and push Docker image for ${{ matrix.os_variant }} + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile_path }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + secrets: | + pipconf=${{ secrets.PIP_CONF_CONTENT }} # Pass pip.conf content as a secret + build-args: | + ANSIBLE_VERSION=${{ matrix.ansible_version }} + ANSIBLE_CORE_VERSION=${{ matrix.ansible_core_version }} + POETRY_INSTALLER_VERSION=${{ matrix.poetry_installer_version }} \ No newline at end of file diff --git a/tests/container_images/docker/centos_7/Dockerfile b/tests/container_images/docker/centos_7/Dockerfile new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/container_images/docker/redhat_9/Dockerfile b/tests/container_images/docker/redhat_9/Dockerfile new file mode 100644 index 0000000000..04ff42a462 --- /dev/null +++ b/tests/container_images/docker/redhat_9/Dockerfile @@ -0,0 +1,98 @@ +# syntax=docker/dockerfile:1.4 +FROM redhat/ubi9 + +# Use bash with pipefail for robust scripting +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Set environment variables for Poetry +ENV PATH="/root/.local/bin:$PATH" + +# Define ARGs for versions (important for cache invalidation and clarity) +ARG ANSIBLE_VERSION="6.1.0" +ARG ANSIBLE_CORE_VERSION="2.13.13" +ARG POETRY_INSTALLER_VERSION="1.5.1" +ARG KUSTOMIZE_VER=5.3.0 + +# 1. Install common system dependencies first +RUN dnf install -y --allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ + dnf install -y --allowerasing \ + dnf-utils \ + curl \ + gnupg2 \ + make \ + gcc \ + parallel \ + sshpass \ + git \ + jq \ + ca-certificates \ + systemd \ + systemd-udev \ + dbus \ + && dnf clean all + +# 2. Install Python versions +# UBI9 default python3 is 3.9 is available via modules. +# Python 3.7 and 3.12 are not directly available in default repos for RHEL9/UBI9. +RUN dnf install -y --allowerasing \ + python3.9 \ + python3.9-devel \ + python3.9-pip \ + && dnf clean all + +# 4. Prepare python venv and install tools +RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ + # Create virtual environments in a common location + python3.9 -m venv /opt/.venv3.9 && \ + curl -sSL https://install.python-poetry.org | /opt/.venv3.9/bin/python - --version "${POETRY_INSTALLER_VERSION}" + +# 5. Install Ansible into the /opt/.venv3.9 virtual environment +RUN /opt/.venv3.9/bin/pip install --no-cache-dir \ + ansible=="${ANSIBLE_VERSION}" \ + ansible-core=="${ANSIBLE_CORE_VERSION}" \ + docker \ + pywinrm + +# Add the /opt/.venv3.9/bin to PATH so 'ansible-playbook' etc. are found +ENV PATH="/opt/.venv3.9/bin:$PATH" + +# 6. Install Docker +# Add Docker's official repository for CentOS/RHEL +RUN dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ + dnf install -y --allowerasing \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + && dnf clean all + +# Add a 'docker' group (though in CI, often run as root) +RUN groupadd docker || true + +# 7. Install Podman (usually pre-installed or easily available) +RUN dnf install -y --allowerasing podman && \ + dnf clean all + +# Clean up any temporary files or caches created during the build to reduce image size +RUN dnf autoremove -y && \ + dnf clean all && \ + rm -rf /tmp/* /var/tmp/* + + +RUN systemctl mask systemd-logind.service \ + systemd-udevd.service \ + systemd-journald.service \ + systemd-resolved.service \ + networkd.service \ + getty@.service \ + graphical.target \ + multi-user.target \ + plymouth-quit-wait.service \ + plymouth-start.service \ + systemd-update-utmp-runlevel.service \ + systemd-update-utmp.service \ + systemd-vconsole-setup.service \ + console-getty.service || true + +CMD ["/usr/sbin/init"] \ No newline at end of file diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile new file mode 100644 index 0000000000..8179e7609c --- /dev/null +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -0,0 +1,125 @@ +FROM ubuntu:24.04 + +# Use bash with pipefail for robust scripting +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Set environment variables for non-interactive apt and Poetry +ENV DEBIAN_FRONTEND=noninteractive +# Poetry installer will put its binaries here, ensure it's in PATH +ENV PATH="/root/.local/bin:$PATH" + +# Define ARGs for versions (important for cache invalidation and clarity) +ARG ANSIBLE_VERSION="6.1.0" # Or specify ansible-core version +ARG ANSIBLE_CORE_VERSION="2.13.13" # Based on your previous output +ARG POETRY_INSTALLER_VERSION="1.5.1" # For the global poetry install +ARG MICROK8S_CHANNEL="1.28/stable" # Specify MicroK8s channel + +# 1. Install common system dependencies first +# This helps consolidate apt updates and reduce image layers +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + curl \ + gnupg \ + lsb-release \ + build-essential \ + parallel \ + sshpass \ + git \ + jq \ + # For Docker installation + ca-certificates \ + # For MicroK8s + snapd \ + systemd \ + systemd-udev \ + dbus \ + # Clean up apt cache to keep image small + && rm -rf /var/lib/apt/lists/* + +# 2. Install Python versions +# Use deadsnakes PPA for older Python versions on newer Ubuntu +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update -y && \ + apt-get install -y --no-install-recommends \ + python3.7 python3.7-venv python3.7-dev \ + python3.9 python3.9-venv python3.9-dev \ + python3.12 python3.12-venv python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Ensure python3 points to 3.12 (default on Ubuntu 24.04) and pip3 to its pip +# This makes 'python3' and 'pip3' commands consistent +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 \ + && update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 + +# 4. Prepare python venv and install tools orca and cloudctl +# Using a single RUN layer for efficiency and to manage secrets +RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ + # Create virtual environments in a common location + python3.12 -m venv /opt/.venv3.12 && \ + python3.9 -m venv /opt/.venv3.9 && \ + python3.7 -m venv /opt/.venv3.7 && \ + \ + # Install Poetry globally using its installer (for default Python 3.12) + # This ensures 'poetry' command is available without venv activation + curl -sSL https://install.python-poetry.org | python3.12 - --version "${POETRY_INSTALLER_VERSION}" + +# 5. Install Ansible globally for the default python3 (3.12) +RUN /opt/.venv3.9/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm + +# 6. Install Docker +# Add Docker's official GPG key +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + chmod a+r /etc/apt/keyrings/docker.gpg + +# Add the Docker repository to Apt sources +RUN echo \ + "deb [arch=\"$(dpkg --print-architecture)\" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + \"$(. /etc/os-release && echo \"$VERSION_CODENAME\")\" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install Docker packages +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + && rm -rf /var/lib/apt/lists/* + +# Add a 'docker' group to prevent permission issues (though in CI, often run as root) +RUN groupadd docker || true + +# 7. Install Podman (Available via apt on Ubuntu) +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + podman \ + && rm -rf /var/lib/apt/lists/* + +# Clean up any temporary files or caches created during the build to reduce image size +RUN apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Set default command or entrypoint if desired for interactive use +# ENTRYPOINT ["/bin/bash"] +# CMD ["/bin/bash"] + +RUN systemctl mask systemd-logind.service \ + systemd-udevd.service \ + systemd-journald.service \ + systemd-resolved.service \ + networkd.service \ + getty@.service \ + graphical.target \ + multi-user.target \ + plymouth-quit-wait.service \ + plymouth-start.service \ + systemd-update-utmp-runlevel.service \ + systemd-update-utmp.service \ + systemd-vconsole-setup.service \ + console-getty.service || true + +CMD ["/usr/sbin/init"] \ No newline at end of file From 258bedc2eb7b9079d08eb669240d66771956a62a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 17 Sep 2025 17:14:08 +0530 Subject: [PATCH 48/91] chore: made changes to ubuntu image --- .github/workflows/ci-env-images.yaml | 2 +- tests/container_images/docker/ubuntu_24/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index 9a16721252..67cbc7f76f 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false # Set to false to build all images even if one fails matrix: - os_variant: [ubuntu, rhel, centos] + os_variant: [ubuntu, rhel] include: - os_variant: ubuntu dockerfile_path: tests/container_images/docker/ubuntu_24/Dockerfile diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile index 8179e7609c..69b71f27e0 100644 --- a/tests/container_images/docker/ubuntu_24/Dockerfile +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -32,7 +32,6 @@ RUN apt-get update -y && \ # For MicroK8s snapd \ systemd \ - systemd-udev \ dbus \ # Clean up apt cache to keep image small && rm -rf /var/lib/apt/lists/* @@ -65,7 +64,7 @@ RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ curl -sSL https://install.python-poetry.org | python3.12 - --version "${POETRY_INSTALLER_VERSION}" # 5. Install Ansible globally for the default python3 (3.12) -RUN /opt/.venv3.9/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm +RUN /opt/.venv3.9/bin/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm # 6. Install Docker # Add Docker's official GPG key From ccf5592ffe03d112bccea6a0a750f78e75cb9708 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 17 Sep 2025 17:29:19 +0530 Subject: [PATCH 49/91] chore: test with dind --- .github/workflows/ci-main.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 45dde98abf..18561b0b37 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -183,12 +183,11 @@ jobs: matrix: # Use official base images for the OSes container_os_image: - - ubuntu:24.04 - - dokken/centos-stream-9 - - redhat/ubi10 + - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-ubuntu + - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel container: image: ${{ matrix.container_os_image }} - options: --privileged -v /var/run/docker.sock:/var/run/docker.sock + options: --rm -tid --privileged --network dind-network -e DOCKER_HOST=tcp://63296e5c590e:2375 --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw runs-on: ubuntu-latest needs: - meta @@ -206,7 +205,15 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - + dind-daemon: + image: docker:dind + options: >- + --privileged + --storage-driver overlay2 + --host=tcp://0.0.0.0:2375 # Listen on 2375 for non-TLS connections + --tls=false # Explicitly disable TLS for the daemon + env: + DOCKER_TLS_CERTDIR: "" steps: - name: Debug -2 run: | From 7fa77f14ba11bafa9e88069d3828d8983e327151 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 17 Sep 2025 17:41:23 +0530 Subject: [PATCH 50/91] chore: test --- .github/workflows/ci-env-images.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index 67cbc7f76f..53c25ff2b3 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -55,8 +55,7 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }} tags: | - type=ref,event=pr,suffix=-pr{{pr.number}} # e.g., ci-env-ubuntu:pr-123 - type=sha,suffix=-{{sha}} # e.g., ci-env-ubuntu:abcdef123456 + latest - name: Build and push Docker image for ${{ matrix.os_variant }} uses: docker/build-push-action@v6 From 08945b0eac2de747bbbe227eb3d3a9409a141041 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 17 Sep 2025 17:42:33 +0530 Subject: [PATCH 51/91] chore: test --- .github/workflows/ci-env-images.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index 53c25ff2b3..b4767aa05a 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -55,7 +55,8 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }} tags: | - latest + type=ref,event=pr,suffix=-pr{{pr.number}} # e.g., ci-env-ubuntu:pr-123 + type=sha,suffix=-{{sha}} # e.g., ci-env-ubuntu:abcdef123456 - name: Build and push Docker image for ${{ matrix.os_variant }} uses: docker/build-push-action@v6 @@ -63,7 +64,7 @@ jobs: context: . file: ${{ matrix.dockerfile_path }} push: true - tags: ${{ steps.meta.outputs.tags }} + tags: latest labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max From 9a66eaa6ef875a6dd4d8d5b7d80cbaa388c76c27 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 10:55:58 +0530 Subject: [PATCH 52/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 18561b0b37..4ca56bdad5 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -187,7 +187,7 @@ jobs: - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel container: image: ${{ matrix.container_os_image }} - options: --rm -tid --privileged --network dind-network -e DOCKER_HOST=tcp://63296e5c590e:2375 --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw + options: --rm -ti --privileged --network dind-network -e DOCKER_HOST=tcp://63296e5c590e:2375 --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw runs-on: ubuntu-latest needs: - meta From ba881a419e8bd6b9c9b20663a1a958e2c1bd17e6 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 11:17:25 +0530 Subject: [PATCH 53/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 4ca56bdad5..501db56cad 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -187,7 +187,7 @@ jobs: - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel container: image: ${{ matrix.container_os_image }} - options: --rm -ti --privileged --network dind-network -e DOCKER_HOST=tcp://63296e5c590e:2375 --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw + options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw runs-on: ubuntu-latest needs: - meta From 7f0ca9a45ad3db6cb8e46c401fb16a1644a988a8 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 11:21:47 +0530 Subject: [PATCH 54/91] chore: test --- .github/workflows/ci-main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 501db56cad..eab7cac5e2 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -209,7 +209,6 @@ jobs: image: docker:dind options: >- --privileged - --storage-driver overlay2 --host=tcp://0.0.0.0:2375 # Listen on 2375 for non-TLS connections --tls=false # Explicitly disable TLS for the daemon env: From 74aad4abd4dcdd1a1ad51e202d88b72ceafd7e10 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 11:29:36 +0530 Subject: [PATCH 55/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index eab7cac5e2..0333aadee5 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -209,6 +209,7 @@ jobs: image: docker:dind options: >- --privileged + --entrypoint dockerd --host=tcp://0.0.0.0:2375 # Listen on 2375 for non-TLS connections --tls=false # Explicitly disable TLS for the daemon env: From 7e0951c20d74910d5caca97af389cf9764183f6f Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 11:43:12 +0530 Subject: [PATCH 56/91] chore: test --- .github/workflows/ci-main.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 0333aadee5..1ce2daf50b 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -207,11 +207,8 @@ jobs: SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz dind-daemon: image: docker:dind - options: >- - --privileged - --entrypoint dockerd - --host=tcp://0.0.0.0:2375 # Listen on 2375 for non-TLS connections - --tls=false # Explicitly disable TLS for the daemon + options: --privileged + command: ["dockerd", "--storage-driver", "overlay2", "--host", "tcp://0.0.0.0:2375", "--tls=false"] env: DOCKER_TLS_CERTDIR: "" steps: From 6ce193a357c47cd7db3aea8da9ab6385643f1b00 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 15:28:20 +0530 Subject: [PATCH 57/91] chore: test --- .github/workflows/ci-main.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 1ce2daf50b..a3eba5cc02 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -207,8 +207,9 @@ jobs: SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz dind-daemon: image: docker:dind - options: --privileged - command: ["dockerd", "--storage-driver", "overlay2", "--host", "tcp://0.0.0.0:2375", "--tls=false"] + options: --privileged --tls=false + ports: + - 2375:2375 env: DOCKER_TLS_CERTDIR: "" steps: From 230a1ab90a955bbc7aeb26f3b26774c21214b60b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 15:34:41 +0530 Subject: [PATCH 58/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index a3eba5cc02..cd8fd3a8c7 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -207,7 +207,7 @@ jobs: SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz dind-daemon: image: docker:dind - options: --privileged --tls=false + options: --privileged ports: - 2375:2375 env: From d7a9825b16d4e1e3b961548ff1cb384666ff9c74 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 18 Sep 2025 15:43:37 +0530 Subject: [PATCH 59/91] chore: test --- .github/workflows/ci-main.yaml | 36 ---------------------------------- 1 file changed, 36 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index cd8fd3a8c7..68a43c35bb 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -217,31 +217,6 @@ jobs: run: | cat /etc/os-release uname -a - - name: Install system dependencies - run: | - if [ -f /etc/debian_version ]; then - echo "Detected Debian/Ubuntu-based OS: ${{ matrix.container_os_image }}" - apt update -y - apt install --no-install-recommends -y python3-pip python3-docker python3-jsondiff build-essential curl gnupg # Added gnupg - apt clean - elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then - echo "Detected RHEL/CentOS-based OS: ${{ matrix.container_os_image }}" - yum update -y || dnf update -y - # Added gnupg2 for RHEL/CentOS systems - yum install -y python3-pip make gcc curl gnupg2 || dnf install -y python3-pip make gcc curl gnupg2 - yum clean all || dnf clean all - else - echo "Unsupported OS detected inside container for ${{ matrix.container_os_image }}. Please add installation commands." - exit 1 - fi - - uses: actions/setup-python@v5 - with: - python-version: 3.9.23 - - name: Create and activate virtual environment - run: | - python -m venv .venv # Creates a virtual environment named '.venv' - echo "VIRTUAL_ENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV # Store path for later use - echo "PATH=$(pwd)/.venv/bin:$PATH" >> $GITHUB_ENV # Add venv bin to PATH for subsequent steps - name: Checkout uses: actions/checkout@v4 with: @@ -251,17 +226,6 @@ jobs: run: | cat /etc/os-release uname -a - - name: Install Ansible and other dependencies as python package - run: | - python -m pip install ansible~=6.1.0 --no-cache-dir \ - && python -m pip install pywinrm>=0.4.2 --no-cache-dir \ - && python -m pip install ansible-lint>=6.0.0 --no-cache-dir - # - name: Installing system dependencies for Ansible - # run: | - # apt update -y \ - # && apt install --no-install-recommends -y build-essential parallel \ - # && apt install --no-install-recommends -y sshpass \ - # && apt clean - name: Configure Ansible Environment Variables env: ANSIBLE_CONFIG: ansible.cfg From 141b159fbe5a8dad052509f7bd8d90fad81f9122 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 19 Sep 2025 11:09:35 +0530 Subject: [PATCH 60/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- ansible/playbooks/docker.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 68a43c35bb..c09b8e46ef 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -263,7 +263,7 @@ jobs: # docker ps - name: Run Ansible Playbook run: | - ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + /opt/.venv3.9/bin/ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml - name: Debug 2 run: | whoami diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index 655405449b..cfe69d6c57 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -4,7 +4,13 @@ vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: + - name: Check if 'docker' command exists on target (if not using docker connection) + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true - name: Docker installation role include_role: name: install_docker + when: + - docker_command_check.rc != 0 - include_tasks: ../tasks/docker/deploy_app.yml From b7d33d694179f8d8332936325fb5bd02895405c2 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 19 Sep 2025 16:35:00 +0530 Subject: [PATCH 61/91] chore: test --- .github/workflows/ci-env-images.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index b4767aa05a..edebe61908 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -43,6 +43,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USER }} # Your Docker Hub username + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: From 0944235ce66a9ca7705048a45ec703fcd0cdbe14 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 19 Sep 2025 16:36:02 +0530 Subject: [PATCH 62/91] chore : test2 --- tests/container_images/docker/redhat_9/Dockerfile | 3 ++- tests/container_images/docker/ubuntu_24/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/container_images/docker/redhat_9/Dockerfile b/tests/container_images/docker/redhat_9/Dockerfile index 04ff42a462..8d5e916459 100644 --- a/tests/container_images/docker/redhat_9/Dockerfile +++ b/tests/container_images/docker/redhat_9/Dockerfile @@ -51,7 +51,8 @@ RUN /opt/.venv3.9/bin/pip install --no-cache-dir \ ansible=="${ANSIBLE_VERSION}" \ ansible-core=="${ANSIBLE_CORE_VERSION}" \ docker \ - pywinrm + pywinrm \ + requests # Add the /opt/.venv3.9/bin to PATH so 'ansible-playbook' etc. are found ENV PATH="/opt/.venv3.9/bin:$PATH" diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile index 69b71f27e0..d8c295290e 100644 --- a/tests/container_images/docker/ubuntu_24/Dockerfile +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -64,7 +64,7 @@ RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ curl -sSL https://install.python-poetry.org | python3.12 - --version "${POETRY_INSTALLER_VERSION}" # 5. Install Ansible globally for the default python3 (3.12) -RUN /opt/.venv3.9/bin/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm +RUN /opt/.venv3.9/bin/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm requests # 6. Install Docker # Add Docker's official GPG key From e65981a075fe8dd3b1d8b5cf014ea19081f1530a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Fri, 19 Sep 2025 17:05:51 +0530 Subject: [PATCH 63/91] chore: test --- .github/workflows/ci-env-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index edebe61908..447f4cfb6e 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -70,7 +70,7 @@ jobs: context: . file: ${{ matrix.dockerfile_path }} push: true - tags: latest + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max From aa542e80842bdae4a701fecf16226fa42d40b31c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 11:28:55 +0530 Subject: [PATCH 64/91] chore: test --- .github/workflows/ci-env-images.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-env-images.yaml b/.github/workflows/ci-env-images.yaml index 447f4cfb6e..bad8507ce6 100644 --- a/.github/workflows/ci-env-images.yaml +++ b/.github/workflows/ci-env-images.yaml @@ -61,8 +61,7 @@ jobs: with: images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }} tags: | - type=ref,event=pr,suffix=-pr{{pr.number}} # e.g., ci-env-ubuntu:pr-123 - type=sha,suffix=-{{sha}} # e.g., ci-env-ubuntu:abcdef123456 + type=raw,value=latest - name: Build and push Docker image for ${{ matrix.os_variant }} uses: docker/build-push-action@v6 From 8a722465c3a7389ba8beb7d73b04761ec8047a48 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 11:59:41 +0530 Subject: [PATCH 65/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c09b8e46ef..335a253543 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -263,7 +263,7 @@ jobs: # docker ps - name: Run Ansible Playbook run: | - /opt/.venv3.9/bin/ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + /opt/.venv3.9/bin/ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml -e 'ansible_python_interpreter=/opt/.venv3.9/bin/python3' - name: Debug 2 run: | whoami From e056a4f7a240b38a8fb695d14c6eaef66faf99f3 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 12:10:04 +0530 Subject: [PATCH 66/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 335a253543..97b229b046 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -260,6 +260,7 @@ jobs: cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file + systemctl status # docker ps - name: Run Ansible Playbook run: | From 8758a2da3edb7451163920966b545104e51e48db Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 12:25:02 +0530 Subject: [PATCH 67/91] chore: test --- .github/workflows/ci-main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 97b229b046..cb35949ef7 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -187,7 +187,9 @@ jobs: - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel container: image: ${{ matrix.container_os_image }} - options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw + options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw + env: + DOCKER_HOST: tcp://dind-daemon:2375 runs-on: ubuntu-latest needs: - meta From 5faf05638804085cac7afa3fe2fb37c268e85517 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 12:36:20 +0530 Subject: [PATCH 68/91] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index cb35949ef7..5851973147 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -189,7 +189,7 @@ jobs: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw env: - DOCKER_HOST: tcp://dind-daemon:2375 + DOCKER_HOST: tcp://localhost:2375 runs-on: ubuntu-latest needs: - meta From 26a1138a9cb44df71092cab1595983c874be653a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 12:45:54 +0530 Subject: [PATCH 69/91] chore : test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5851973147..807122ae9f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -189,7 +189,7 @@ jobs: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw env: - DOCKER_HOST: tcp://localhost:2375 + DOCKER_HOST: tcp://0.0.0.0:2375 runs-on: ubuntu-latest needs: - meta From 6669c1fda6daad195106e80ce9636af096e4612d Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 12:56:13 +0530 Subject: [PATCH 70/91] chore: t --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 807122ae9f..e03e5748f3 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -189,7 +189,7 @@ jobs: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw env: - DOCKER_HOST: tcp://0.0.0.0:2375 + DOCKER_HOST: tcp://docker:2375 runs-on: ubuntu-latest needs: - meta From 8d76ac3ba45e68acfb3710b2d9352e1580bd58fd Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 13:25:38 +0530 Subject: [PATCH 71/91] chore : test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e03e5748f3..cb35949ef7 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -189,7 +189,7 @@ jobs: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw env: - DOCKER_HOST: tcp://docker:2375 + DOCKER_HOST: tcp://dind-daemon:2375 runs-on: ubuntu-latest needs: - meta From ed95f5b5822988beb6e1ade24f7985840f12cd3a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 13:53:05 +0530 Subject: [PATCH 72/91] chore : test --- .github/workflows/ci-main.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index cb35949ef7..496bccb20e 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -217,6 +217,22 @@ jobs: steps: - name: Debug -2 run: | + echo "--- Main Container's IP ---" + ip a | grep 'inet ' | grep -v 127.0.0.1 + echo "--- Main Container's resolv.conf ---" + cat /etc/resolv.conf + echo "--- Ping dind-daemon by name ---" + ping -c 3 dind-daemon || echo "Ping to dind-daemon failed" + echo "--- Ping dind-daemon by IP (if known) ---" + # Try to get the IP of dind-daemon from the host's perspective if possible + # This is hard from within the container without mounting docker.sock + # For now, just check name resolution. + echo "--- Check DNS resolution of dind-daemon ---" + getent hosts dind-daemon || echo "getent hosts dind-daemon failed" + echo "--- Test connection to dind-daemon:2375 ---" + # Use netcat or similar to test raw TCP connection + apt-get update && apt-get install -y netcat-traditional # Install netcat if not present + nc -zv dind-daemon 2375 || echo "Netcat connection to dind-daemon:2375 failed" cat /etc/os-release uname -a - name: Checkout From e25ed76b37ef2736d80f03dc830cdeb76fea7baf Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 14:05:28 +0530 Subject: [PATCH 73/91] chore: test --- .github/workflows/ci-main.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 496bccb20e..71e8732561 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -187,9 +187,7 @@ jobs: - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel container: image: ${{ matrix.container_os_image }} - options: --privileged --cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw - env: - DOCKER_HOST: tcp://dind-daemon:2375 + options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw runs-on: ubuntu-latest needs: - meta From d151e3dc797cb7aaf2df1ecf050762ef054ba92e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 16:12:29 +0530 Subject: [PATCH 74/91] chore: test --- tests/container_images/docker/redhat_9/Dockerfile | 2 +- tests/container_images/docker/ubuntu_24/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/container_images/docker/redhat_9/Dockerfile b/tests/container_images/docker/redhat_9/Dockerfile index 8d5e916459..a8adf51c3c 100644 --- a/tests/container_images/docker/redhat_9/Dockerfile +++ b/tests/container_images/docker/redhat_9/Dockerfile @@ -96,4 +96,4 @@ RUN systemctl mask systemd-logind.service \ systemd-vconsole-setup.service \ console-getty.service || true -CMD ["/usr/sbin/init"] \ No newline at end of file +ENTRYPOINT ["/usr/sbin/init"] \ No newline at end of file diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile index d8c295290e..a0757120ea 100644 --- a/tests/container_images/docker/ubuntu_24/Dockerfile +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -121,4 +121,4 @@ RUN systemctl mask systemd-logind.service \ systemd-vconsole-setup.service \ console-getty.service || true -CMD ["/usr/sbin/init"] \ No newline at end of file +ENTRYPOINT ["/usr/sbin/init"] \ No newline at end of file From e8d8dd592e49d4291321edaaef9e902b2fcd9c43 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 16:21:23 +0530 Subject: [PATCH 75/91] yy --- .github/workflows/ci-main.yaml | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 71e8732561..88045e1417 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -213,26 +213,26 @@ jobs: env: DOCKER_TLS_CERTDIR: "" steps: - - name: Debug -2 - run: | - echo "--- Main Container's IP ---" - ip a | grep 'inet ' | grep -v 127.0.0.1 - echo "--- Main Container's resolv.conf ---" - cat /etc/resolv.conf - echo "--- Ping dind-daemon by name ---" - ping -c 3 dind-daemon || echo "Ping to dind-daemon failed" - echo "--- Ping dind-daemon by IP (if known) ---" - # Try to get the IP of dind-daemon from the host's perspective if possible - # This is hard from within the container without mounting docker.sock - # For now, just check name resolution. - echo "--- Check DNS resolution of dind-daemon ---" - getent hosts dind-daemon || echo "getent hosts dind-daemon failed" - echo "--- Test connection to dind-daemon:2375 ---" - # Use netcat or similar to test raw TCP connection - apt-get update && apt-get install -y netcat-traditional # Install netcat if not present - nc -zv dind-daemon 2375 || echo "Netcat connection to dind-daemon:2375 failed" - cat /etc/os-release - uname -a + # - name: Debug -2 + # run: | + # echo "--- Main Container's IP ---" + # ip a | grep 'inet ' | grep -v 127.0.0.1 + # echo "--- Main Container's resolv.conf ---" + # cat /etc/resolv.conf + # echo "--- Ping dind-daemon by name ---" + # ping -c 3 dind-daemon || echo "Ping to dind-daemon failed" + # echo "--- Ping dind-daemon by IP (if known) ---" + # # Try to get the IP of dind-daemon from the host's perspective if possible + # # This is hard from within the container without mounting docker.sock + # # For now, just check name resolution. + # echo "--- Check DNS resolution of dind-daemon ---" + # getent hosts dind-daemon || echo "getent hosts dind-daemon failed" + # echo "--- Test connection to dind-daemon:2375 ---" + # # Use netcat or similar to test raw TCP connection + # apt-get update && apt-get install -y netcat-traditional # Install netcat if not present + # nc -zv dind-daemon 2375 || echo "Netcat connection to dind-daemon:2375 failed" + # cat /etc/os-release + # uname -a - name: Checkout uses: actions/checkout@v4 with: From 8f9cd027d842c45ebb2418b595a2ad2799e72f8b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 18:24:42 +0530 Subject: [PATCH 76/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 88045e1417..196cc7fc0e 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -188,6 +188,7 @@ jobs: container: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw + entrypoint: ["/usr/sbin/init"] runs-on: ubuntu-latest needs: - meta From 634fdccb89b63cfb27d61a8a4da60ef8479d6e30 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 22 Sep 2025 18:55:49 +0530 Subject: [PATCH 77/91] chore: test --- .github/workflows/ci-main.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 196cc7fc0e..f8319f207d 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -188,7 +188,6 @@ jobs: container: image: ${{ matrix.container_os_image }} options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw - entrypoint: ["/usr/sbin/init"] runs-on: ubuntu-latest needs: - meta @@ -277,7 +276,6 @@ jobs: cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file - systemctl status # docker ps - name: Run Ansible Playbook run: | @@ -287,7 +285,7 @@ jobs: whoami uname -a docker ps - systemctl status sc4s + systemctl --user status sc4s docker images - name: Run tests run: | From 69a77f608153bf733ad07b504b6747667b63a8dc Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 11:33:51 +0530 Subject: [PATCH 78/91] chore: test --- ansible/tasks/docker/deploy_app.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 6e090fa1c9..d59cec5291 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -36,3 +36,4 @@ enabled: true daemon_reload: true state: started + scope: user From b086907f7d03bef411a7502b870a34b5d6fe0cc5 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 11:53:27 +0530 Subject: [PATCH 79/91] chore: test --- .github/workflows/ci-main.yaml | 1 + .../docker/ubuntu_24/Dockerfile | 132 ++---------------- 2 files changed, 15 insertions(+), 118 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index f8319f207d..6030c7263e 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -240,6 +240,7 @@ jobs: persist-credentials: false - name: Debug -1 run: | + systemctl status cat /etc/os-release uname -a - name: Configure Ansible Environment Variables diff --git a/tests/container_images/docker/ubuntu_24/Dockerfile b/tests/container_images/docker/ubuntu_24/Dockerfile index a0757120ea..af793b8ad5 100644 --- a/tests/container_images/docker/ubuntu_24/Dockerfile +++ b/tests/container_images/docker/ubuntu_24/Dockerfile @@ -1,124 +1,20 @@ FROM ubuntu:24.04 -# Use bash with pipefail for robust scripting -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV container=docker \ + DEBIAN_FRONTEND=noninteractive -# Set environment variables for non-interactive apt and Poetry -ENV DEBIAN_FRONTEND=noninteractive -# Poetry installer will put its binaries here, ensure it's in PATH -ENV PATH="/root/.local/bin:$PATH" +RUN INSTALL_PKGS='findutils iproute2 python3 python3-apt sudo systemd' \ + && apt-get update && apt-get install $INSTALL_PKGS -y --no-install-recommends \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Define ARGs for versions (important for cache invalidation and clarity) -ARG ANSIBLE_VERSION="6.1.0" # Or specify ansible-core version -ARG ANSIBLE_CORE_VERSION="2.13.13" # Based on your previous output -ARG POETRY_INSTALLER_VERSION="1.5.1" # For the global poetry install -ARG MICROK8S_CHANNEL="1.28/stable" # Specify MicroK8s channel +RUN find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -print0 | xargs -0 rm -vf -# 1. Install common system dependencies first -# This helps consolidate apt updates and reduce image layers -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - curl \ - gnupg \ - lsb-release \ - build-essential \ - parallel \ - sshpass \ - git \ - jq \ - # For Docker installation - ca-certificates \ - # For MicroK8s - snapd \ - systemd \ - dbus \ - # Clean up apt cache to keep image small - && rm -rf /var/lib/apt/lists/* +VOLUME [ "/sys/fs/cgroup" ] -# 2. Install Python versions -# Use deadsnakes PPA for older Python versions on newer Ubuntu -RUN add-apt-repository -y ppa:deadsnakes/ppa && \ - apt-get update -y && \ - apt-get install -y --no-install-recommends \ - python3.7 python3.7-venv python3.7-dev \ - python3.9 python3.9-venv python3.9-dev \ - python3.12 python3.12-venv python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Ensure python3 points to 3.12 (default on Ubuntu 24.04) and pip3 to its pip -# This makes 'python3' and 'pip3' commands consistent -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 \ - && update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.12 100 - -# 4. Prepare python venv and install tools orca and cloudctl -# Using a single RUN layer for efficiency and to manage secrets -RUN --mount=type=secret,id=pipconf,dst=/root/.pip/pip.conf \ - # Create virtual environments in a common location - python3.12 -m venv /opt/.venv3.12 && \ - python3.9 -m venv /opt/.venv3.9 && \ - python3.7 -m venv /opt/.venv3.7 && \ - \ - # Install Poetry globally using its installer (for default Python 3.12) - # This ensures 'poetry' command is available without venv activation - curl -sSL https://install.python-poetry.org | python3.12 - --version "${POETRY_INSTALLER_VERSION}" - -# 5. Install Ansible globally for the default python3 (3.12) -RUN /opt/.venv3.9/bin/pip3 install ansible=="${ANSIBLE_VERSION}" ansible-core=="${ANSIBLE_CORE_VERSION}" docker pywinrm requests - -# 6. Install Docker -# Add Docker's official GPG key -RUN install -m 0755 -d /etc/apt/keyrings && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ - chmod a+r /etc/apt/keyrings/docker.gpg - -# Add the Docker repository to Apt sources -RUN echo \ - "deb [arch=\"$(dpkg --print-architecture)\" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - \"$(. /etc/os-release && echo \"$VERSION_CODENAME\")\" stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Install Docker packages -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - docker-buildx-plugin \ - docker-compose-plugin \ - && rm -rf /var/lib/apt/lists/* - -# Add a 'docker' group to prevent permission issues (though in CI, often run as root) -RUN groupadd docker || true - -# 7. Install Podman (Available via apt on Ubuntu) -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - podman \ - && rm -rf /var/lib/apt/lists/* - -# Clean up any temporary files or caches created during the build to reduce image size -RUN apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Set default command or entrypoint if desired for interactive use -# ENTRYPOINT ["/bin/bash"] -# CMD ["/bin/bash"] - -RUN systemctl mask systemd-logind.service \ - systemd-udevd.service \ - systemd-journald.service \ - systemd-resolved.service \ - networkd.service \ - getty@.service \ - graphical.target \ - multi-user.target \ - plymouth-quit-wait.service \ - plymouth-start.service \ - systemd-update-utmp-runlevel.service \ - systemd-update-utmp.service \ - systemd-vconsole-setup.service \ - console-getty.service || true - -ENTRYPOINT ["/usr/sbin/init"] \ No newline at end of file +ENTRYPOINT [ "/lib/systemd/systemd" ] \ No newline at end of file From 2fc3bd92de038a3fab550fec2afecb87510c90db Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 12:16:47 +0530 Subject: [PATCH 80/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 6030c7263e..367ae4f04c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -240,6 +240,7 @@ jobs: persist-credentials: false - name: Debug -1 run: | + /lib/systemd/systemd systemctl status cat /etc/os-release uname -a From d7e83a56d18d05869874b47f1b5de938a198390e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 12:23:54 +0530 Subject: [PATCH 81/91] chore: test --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 367ae4f04c..7791ac1992 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -240,8 +240,8 @@ jobs: persist-credentials: false - name: Debug -1 run: | - /lib/systemd/systemd - systemctl status + /lib/systemd/systemd --user + systemctl ---user status cat /etc/os-release uname -a - name: Configure Ansible Environment Variables From 684cee883ced6815c8e9256cbcc1ac5c9dff7ee7 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 13:26:02 +0530 Subject: [PATCH 82/91] test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 7791ac1992..8e859527cb 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -240,6 +240,7 @@ jobs: persist-credentials: false - name: Debug -1 run: | + docker ps /lib/systemd/systemd --user systemctl ---user status cat /etc/os-release From bd82c2a73ccc1f456698804c5bb11a5c35ec76d0 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 17:14:03 +0530 Subject: [PATCH 83/91] chore: test 2 --- .github/workflows/ci-main.yaml | 56 ++++++---------------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 8e859527cb..2688b665d4 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,16 +178,6 @@ jobs: scanners: "vuln" test-container: - strategy: - fail-fast: false - matrix: - # Use official base images for the OSes - container_os_image: - - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-ubuntu - - ghcr.io/splunk/splunk-connect-for-syslog/ci-env-rhel - container: - image: ${{ matrix.container_os_image }} - options: --privileged --cgroupns=host -e DOCKER_HOST=tcp://dind-daemon:2375 -v /sys/fs/cgroup:/sys/fs/cgroup:rw runs-on: ubuntu-latest needs: - meta @@ -205,46 +195,18 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - dind-daemon: - image: docker:dind - options: --privileged - ports: - - 2375:2375 - env: - DOCKER_TLS_CERTDIR: "" + steps: - # - name: Debug -2 - # run: | - # echo "--- Main Container's IP ---" - # ip a | grep 'inet ' | grep -v 127.0.0.1 - # echo "--- Main Container's resolv.conf ---" - # cat /etc/resolv.conf - # echo "--- Ping dind-daemon by name ---" - # ping -c 3 dind-daemon || echo "Ping to dind-daemon failed" - # echo "--- Ping dind-daemon by IP (if known) ---" - # # Try to get the IP of dind-daemon from the host's perspective if possible - # # This is hard from within the container without mounting docker.sock - # # For now, just check name resolution. - # echo "--- Check DNS resolution of dind-daemon ---" - # getent hosts dind-daemon || echo "getent hosts dind-daemon failed" - # echo "--- Test connection to dind-daemon:2375 ---" - # # Use netcat or similar to test raw TCP connection - # apt-get update && apt-get install -y netcat-traditional # Install netcat if not present - # nc -zv dind-daemon 2375 || echo "Netcat connection to dind-daemon:2375 failed" - # cat /etc/os-release - # uname -a - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Debug -1 + - name: Install Ansible and other dependencies as python package run: | - docker ps - /lib/systemd/systemd --user - systemctl ---user status - cat /etc/os-release - uname -a + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir - name: Configure Ansible Environment Variables env: ANSIBLE_CONFIG: ansible.cfg @@ -279,16 +241,16 @@ jobs: cat /etc/os-release cat ansible/inventory/inventory.yaml cat ansible/resources/env_file - # docker ps + docker ps - name: Run Ansible Playbook run: | - /opt/.venv3.9/bin/ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml -e 'ansible_python_interpreter=/opt/.venv3.9/bin/python3' + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml - name: Debug 2 run: | whoami uname -a docker ps - systemctl --user status sc4s + systemctl status sc4s docker images - name: Run tests run: | @@ -522,4 +484,4 @@ jobs: @google/semantic-release-replace-plugin@1.2.0 conventional-changelog-conventionalcommits@6.1.0 env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} \ No newline at end of file From 8477a90081e80db5989d80460dfa8778682cc96c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 17:30:15 +0530 Subject: [PATCH 84/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 2688b665d4..678d2ae0bf 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -207,6 +207,7 @@ jobs: pip install ansible~=6.1.0 --no-cache-dir \ && pip install pywinrm>=0.4.2 --no-cache-dir \ && pip install ansible-lint>=6.0.0 --no-cache-dir + && pip install docker - name: Configure Ansible Environment Variables env: ANSIBLE_CONFIG: ansible.cfg From 6e0c9c3fa74d1db404fb2beb0417aaf94e619c46 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 18:12:41 +0530 Subject: [PATCH 85/91] 1 --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 678d2ae0bf..01c0de8f7c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -206,7 +206,7 @@ jobs: run: | pip install ansible~=6.1.0 --no-cache-dir \ && pip install pywinrm>=0.4.2 --no-cache-dir \ - && pip install ansible-lint>=6.0.0 --no-cache-dir + && pip install ansible-lint>=6.0.0 --no-cache-dir \ && pip install docker - name: Configure Ansible Environment Variables env: From a160ba57afbee2fcc1ec3bf6cc109ae1feeafb12 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 23 Sep 2025 19:18:55 +0530 Subject: [PATCH 86/91] chore: test --- ansible/playbooks/docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index cfe69d6c57..acf6ad215a 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -11,6 +11,4 @@ - name: Docker installation role include_role: name: install_docker - when: - - docker_command_check.rc != 0 - include_tasks: ../tasks/docker/deploy_app.yml From 1bfa60066d936dd5b3d132701b054efaeac333e5 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 24 Sep 2025 11:59:49 +0530 Subject: [PATCH 87/91] chore: test --- ansible/tasks/docker/deploy_app.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index d59cec5291..6e090fa1c9 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -36,4 +36,3 @@ enabled: true daemon_reload: true state: started - scope: user From 6d2e9f455267dcb092934b75911fb31354d5b689 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 24 Sep 2025 12:19:55 +0530 Subject: [PATCH 88/91] chore: test --- .github/workflows/ci-main.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 01c0de8f7c..3820917298 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,6 +178,9 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest needs: - meta @@ -245,7 +248,7 @@ jobs: docker ps - name: Run Ansible Playbook run: | - ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/docker.yml + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Debug 2 run: | whoami From 59a0550e7be228d274fd901e642667cb8163000a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 24 Sep 2025 13:25:55 +0530 Subject: [PATCH 89/91] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 3820917298..90621d9873 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -230,6 +230,7 @@ jobs: node_1: ansible_host: 127.0.0.1 ansible_connection: local + ansible_user: root - name: Update env_file run: | echo "Updating ansible/inventory/inventory.yaml" From 0b77aa6f81f9745de6faa1b9621fd88820bf78b3 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 24 Sep 2025 13:44:18 +0530 Subject: [PATCH 90/91] chore: test --- ansible/tasks/docker/deploy_app.yml | 8 ++++---- ansible/tasks/podman/deploy_app.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 6e090fa1c9..85ad163724 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -14,16 +14,16 @@ copy: src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" - owner: "root" - group: "root" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" mode: u=rw,g=rw - name: Copying env_file file on the server copy: src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" - owner: "root" - group: "root" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" mode: u=rw,g=rw - name: Create a volume diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 07632d8eaa..c977370946 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/podman_sc4s.service" + src: "../../../resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" From d6ed73addeddee9bcf3a52d7b2020f115d02bd63 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 24 Sep 2025 13:52:09 +0530 Subject: [PATCH 91/91] chore: test --- ansible/tasks/podman/deploy_app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index c977370946..f74fe85807 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "../../../resources/podman_sc4s.service" + src: "../../../ansible/resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}"