Centos 8 support (#422)

* base centos 8 image
* py23 image centos 8
* py23 image and makefile centos 8
* Support for createdefaults.py py2/py3

Author: nwang92

Makefile

@@ -58,7 +58,7 @@ ansible:
 	@cat splunk-ansible/version.txt
 
 ##### Base images #####
-base: base-debian-9 base-debian-10 base-centos-7 base-redhat-8 base-windows-2016
+base: base-debian-9 base-debian-10 base-centos-7 base-centos-8 base-redhat-8 base-windows-2016
 
 base-debian-10:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-debian-10:${IMAGE_VERSION} ./base/debian-10
@@ -69,14 +69,17 @@ base-debian-9:
 base-centos-7:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-7:${IMAGE_VERSION} ./base/centos-7
 
+base-centos-8:
+	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-8:${IMAGE_VERSION} ./base/centos-8
+
 base-redhat-8:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} --label version=${SPLUNK_VERSION} -t base-redhat-8:${IMAGE_VERSION} ./base/redhat-8
 
 base-windows-2016:
 	docker build ${DOCKER_BUILD_FLAGS} -t base-windows-2016:${IMAGE_VERSION} ./base/windows-2016
 
 ##### Minimal images #####
-minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-redhat-8
+minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-centos-8 minimal-redhat-8
 
 minimal-debian-9: base-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -99,6 +102,13 @@ minimal-centos-7: base-centos-7
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
 		--target minimal -t minimal-centos-7:${IMAGE_VERSION} .	
 
+minimal-centos-8: base-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		--target minimal -t minimal-centos-8:${IMAGE_VERSION} .
+
 minimal-redhat-8: base-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f splunk/common-files/Dockerfile \
@@ -107,7 +117,7 @@ minimal-redhat-8: base-redhat-8
 		--target minimal -t minimal-redhat-8:${IMAGE_VERSION} .
 
 ##### Bare images #####
-bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-redhat-8
+bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-centos-8 bare-redhat-8
 
 bare-debian-9: base-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -128,7 +138,14 @@ bare-centos-7: base-centos-7
 		-f splunk/common-files/Dockerfile \
 		--build-arg SPLUNK_BASE_IMAGE=base-centos-7 \
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
-		--target bare -t bare-centos-7:${IMAGE_VERSION} .	
+		--target bare -t bare-centos-7:${IMAGE_VERSION} .
+
+bare-centos-8: base-centos-7
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		--target bare -t bare-centos-8:${IMAGE_VERSION} .	
 
 bare-redhat-8: base-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -138,7 +155,7 @@ bare-redhat-8: base-redhat-8
 		--target bare -t bare-redhat-8:${IMAGE_VERSION} .
 
 ##### Splunk images #####
-splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8
+splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-centos-8 splunk-redhat-8
 
 splunk-debian-9: base-debian-9 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -161,6 +178,13 @@ splunk-centos-7: base-centos-7 ansible
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
 		-t splunk-centos-7:${IMAGE_VERSION} .
 
+splunk-centos-8: base-centos-8 ansible 
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		-t splunk-centos-8:${IMAGE_VERSION} .
+
 splunk-redhat-8: base-redhat-8 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f splunk/common-files/Dockerfile \
@@ -176,7 +200,7 @@ splunk-windows-2016: base-windows-2016 ansible
 		-t splunk-windows-2016:${IMAGE_VERSION} .
 
 ##### UF images #####
-uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8
+uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-centos-8 uf-redhat-8
 
 ufbare-debian-9: base-debian-9 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -213,6 +237,13 @@ uf-centos-7: base-centos-7 ansible
 		--build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \
 		-t uf-centos-7:${IMAGE_VERSION} .
 
+uf-centos-8: base-centos-8 ansible
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f uf/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \
+		-t uf-centos-8:${IMAGE_VERSION} .
+
 uf-redhat-8: base-redhat-8 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f uf/common-files/Dockerfile \
@@ -229,7 +260,7 @@ uf-windows-2016: base-windows-2016 ansible
 
 
 ##### Python 3 support #####
-splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8
+splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-centos-8 splunk-py23-redhat-8
 
 splunk-py23-debian-9: splunk-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -249,6 +280,12 @@ splunk-py23-centos-7: splunk-centos-7
 		--build-arg SPLUNK_PRODUCT=splunk \
 		-t splunk-py23-centos-7:${IMAGE_VERSION} .
 
+splunk-py23-centos-8: splunk-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f py23-image/centos-8/Dockerfile \
+		--build-arg SPLUNK_PRODUCT=splunk \
+		-t splunk-py23-centos-8:${IMAGE_VERSION} .
+
 splunk-py23-redhat-8: splunk-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f py23-image/redhat-8/Dockerfile \
@@ -275,6 +312,12 @@ uf-py23-centos-7: uf-centos-7
 		--build-arg SPLUNK_PRODUCT=uf \
 		-t uf-py23-centos-7:${IMAGE_VERSION} .
 
+uf-py23-centos-8: uf-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f py23-image/centos-8/Dockerfile \
+		--build-arg SPLUNK_PRODUCT=uf \
+		-t uf-py23-centos-8:${IMAGE_VERSION} .
+
 uf-py23-redhat-8: uf-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f py23-image/redhat-8/Dockerfile \

base/centos-7/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

base/centos-7/install.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

base/centos-8/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2018-2020 Splunk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:8
+LABEL maintainer="support@splunk.com"
+
+ARG SCLOUD_URL
+ENV SCLOUD_URL ${SCLOUD_URL}
+
+COPY install.sh /install.sh
+RUN /install.sh && rm -rf /install.sh

base/centos-8/install.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright 2018-2020 Splunk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+yum -y install glibc-locale-source glibc-langpack-en
+
+localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+export LANG=en_US.utf8
+
+yum -y update && yum -y install wget sudo epel-release make
+yum -y install ansible python3-requests python3-jmespath
+
+# Install busybox
+wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch`
+chmod +x /bin/busybox
+
+# Install scloud
+wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL}
+tar -xf /usr/bin/scloud.tar.gz -C /usr/bin/
+rm /usr/bin/scloud.tar.gz
+
+cd /bin
+ln -s busybox killall
+ln -s busybox netstat
+ln -s busybox nslookup
+ln -s busybox readline
+ln -s busybox route
+ln -s busybox syslogd
+ln -s busybox traceroute
+chmod u+s /bin/ping
+groupadd sudo
+
+echo "
+## Allows people in group sudo to run all commands
+%sudo  ALL=(ALL)       ALL" >> /etc/sudoers
+
+# symlink for python3
+ln -s /bin/python3 /bin/python
+
+# Clean
+yum clean all
+rm -rf /install.sh /anaconda-post.log /var/log/anaconda/*

py23-image/centos-8/Dockerfile

@@ -0,0 +1,29 @@
+ARG SPLUNK_PRODUCT=splunk
+FROM ${SPLUNK_PRODUCT}-centos-8:latest
+USER root
+
+RUN yum -y update
+RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python3-pip python2 python2-pip
+
+# manual installation of python 3.7 as default distro version is 3.6
+RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
+    && tar xzf Python-3.7.4.tgz \
+    && cd Python-3.7.4 \
+    && ./configure --enable-optimizations --prefix=/usr \
+    && make install \
+    && cd .. \
+    && rm Python-3.7.4.tgz \
+    && rm -r Python-3.7.4 \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.7 get-pip.py \
+    && rm -f get-pip.py \
+    # pip version is not automatically "fixed", unlike debian-based
+    && ln -sf /usr/bin/pip2 /usr/bin/pip \
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip3
+    # add python alias
+    # && ln -s /bin/python3 /bin/python
+
+RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \
+    && yum autoremove -y \
+    && yum clean all
+RUN pip3 --no-cache-dir install ansible requests

splunk/common-files/createdefaults.py

@@ -1,5 +1,5 @@
 #! /usr/bin/python
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import six
 import sys
 import uuid
 import random
@@ -32,10 +33,14 @@
 def random_generator(size=24):
     # Use System Random for
     rng = random.SystemRandom()
-    bytes = [chr(rng.randrange(256)) for i in range(size)]
-    s = ''.join(bytes)
+    b = [chr(rng.randrange(256)) for i in range(size)]
+    s = ''.join(b)
+    if six.PY2:
+        s = base64.b64encode(s)
+    else:
+        s = base64.b64encode(s.encode()).decode()
+    return s
 
-    return base64.b64encode(s)
 
 # if there are no environment vars set, lets make some safe defaults
 if not splunk_hec_token:

uf/common-files/createdefaults.py

@@ -1,5 +1,5 @@
 #! /usr/bin/python
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import six
 import sys
 import uuid
 import random
@@ -25,26 +26,40 @@
 splunk_hec_token = os.environ.get("SPLUNK_HEC_TOKEN", None)
 splunk_password = os.environ.get("SPLUNK_PASSWORD", None)
 splunk_idxc_secret = os.environ.get("SPLUNK_IDXC_SECRET", None)
+splunk_idxc_pass4SymmKey = os.environ.get("SPLUNK_IDXC_PASS4SYMMKEY", None)
 splunk_shc_secret = os.environ.get("SPLUNK_SHC_SECRET", None)
+splunk_shc_pass4SymmKey = os.environ.get("SPLUNK_SHC_PASS4SYMMKEY", None)
 
 def random_generator(size=24):
     # Use System Random for
     rng = random.SystemRandom()
-    bytes = [chr(rng.randrange(256)) for i in range(size)]
-    s = ''.join(bytes)
+    b = [chr(rng.randrange(256)) for i in range(size)]
+    s = ''.join(b)
+    if six.PY2:
+        s = base64.b64encode(s)
+    else:
+        s = base64.b64encode(s.encode()).decode()
+    return s
 
-    return base64.b64encode(s)
 
 # if there are no environment vars set, lets make some safe defaults
 if not splunk_hec_token:
     tempuuid=uuid.uuid4()
     os.environ["SPLUNK_HEC_TOKEN"] = str(tempuuid)
 if not splunk_password:
     os.environ["SPLUNK_PASSWORD"] = random_generator()
-if not splunk_idxc_secret:
-    os.environ["SPLUNK_IDXC_SECRET"] = random_generator()
-if not splunk_shc_secret:
-    os.environ["SPLUNK_SHC_SECRET"] = random_generator()
+if splunk_idxc_pass4SymmKey:
+    os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = splunk_idxc_pass4SymmKey
+elif splunk_idxc_secret:
+    os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = splunk_idxc_secret
+else:
+    os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = random_generator()
+if splunk_shc_secret:
+    os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = splunk_shc_pass4SymmKey
+elif splunk_shc_pass4SymmKey:
+    os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = splunk_shc_secret
+else:
+    os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = random_generator()
 sys.argv.append("--write-to-stdout")
 import environ
 environ.main()