Rewrite tests to add parallelization (#402)

* Initial commit

* initial commit

* added more tests

* fix todo

* added more tests

* changed makefile to run new tests

* added tests; removed some container names; working

* remove unnecessary functions and container names

* add env variables for tests to avoid timeout

* fix broken test

* add parallelism

* fix failing tests

* fix spelling error

* adding another thread for tests

* fix flaky test

* add missing tests

* seperate tests by size

* rewrite circleci architecture

* revert architecture changes

* add rerun to help mitigate flaky tests

* seperate tests into single and distributed

* add missing tests

* add removal of default.yml

* Fixing non-terminating loop, more structured OOP

* remove duplicate functions

* remove check_for_default

* remove tar conflicts

* remove test_docker_splunk

Co-authored-by: Nelson Wang <nwang@splunk.com>
Co-authored-by: cameronn <cameronn@splunk.com>
Co-authored-by: TheCamNelson <ccnelson@uwaterloo.com>
Co-authored-by: Nelson Wang <nwang92@users.noreply.github.com>

Author: 5 people

.circleci/config.yml

@@ -61,8 +61,12 @@ jobs:
           name: Test if image size increase
           command: make test_debian10_image_size
       - run:
-          name: Run Debian 10 image tests
-          command: make run_tests_debian10
+          name: Run small Debian 10 image tests
+          command: make run_small_tests_debian10
+          no_output_timeout: 20m
+      - run:
+          name: Run large Debian 10 image tests
+          command: make run_large_tests_debian10
           no_output_timeout: 20m
       - store_artifacts:
           path: test-results
@@ -95,8 +99,12 @@ jobs:
           command: |
             make uf-redhat-8
       - run:
-          name: Run Redhat 8 image tests
-          command: make run_tests_redhat8
+          name: Run small Redhat 8 image tests
+          command: make run_small_tests_redhat8
+          no_output_timeout: 20m
+      - run:
+          name: Run large Redhat 8 image tests
+          command: make run_large_tests_redhat8
           no_output_timeout: 20m
       - store_artifacts:
           path: test-results
@@ -144,6 +152,4 @@ workflows:
       - security-scanning
       - debian10-testing
       - container-validation
-      - redhat8-testing
-
-      
+      - redhat8-testing

.gitignore

@@ -8,5 +8,4 @@ splunk-ansible
 *.xml
 *clair-scanner*
 .env
-tests/fixtures/*
 venv

Makefile

@@ -289,23 +289,35 @@ sample-compose-up: sample-compose-down
 sample-compose-down:
 	docker-compose -f test_scenarios/${SPLUNK_COMPOSE} down --volumes --remove-orphans || true
 
-test: clean ansible test_setup all run_tests_centos7 run_tests_redhat8 run_tests_debian9
+test: clean ansible test_setup all run_small_tests run_large_tests
 
-test_centos7: clean ansible splunk-centos-7 uf-centos-7 test_setup run_tests_centos7
+run_small_tests: run_small_tests_centos7 run_small_tests_redhat8 run_small_tests_debian9 run_small_tests_debian10
 
-test_redhat8: clean ansible splunk-redhat-8 uf-redhat-8 test_setup run_tests_redhat8
+run_large_tests: run_large_tests_centos7 run_large_tests_redhat8 run_large_tests_debian9 run_large_tests_debian10
 
-test_debian9: clean ansible splunk-debian-9 uf-debian-9 test_setup run_tests_debian9
+test_centos7: clean ansible splunk-centos-7 uf-centos-7 test_setup run_small_tests_centos7 run_large_tests_centos7
 
-test_debian10: clean ansible splunk-debian-10 uf-debian-10 test_setup run_tests_debian10
+test_redhat8: clean ansible splunk-redhat-8 uf-redhat-8 test_setup run_small_tests_redhat8 run_large_tests_redhat8
 
-run_tests_centos7:
-	@echo 'Running the super awesome tests; CentOS 7'
-	pytest -sv tests/test_docker_splunk.py --platform centos-7 --junitxml test-results/centos7-result/testresults_centos7.xml
+test_debian9: clean ansible splunk-debian-9 uf-debian-9 test_setup run_small_tests_debian9 run_large_tests_debian9
 
-run_tests_redhat8:
-	@echo 'Running the super awesome tests; RedHat 8'
-	pytest -sv tests/test_docker_splunk.py --platform redhat-8 --junitxml test-results/redhat8-result/testresults_redhat8.xml
+test_debian10: clean ansible splunk-debian-10 uf-debian-10 test_setup run_small_tests_debian10 run_large_tests_debian10
+
+run_small_tests_centos7:
+	@echo 'Running the super awesome small tests; CentOS 7'
+	pytest -n 3 --reruns 1 -sv tests/test_single_splunk_image.py --platform centos-7 --junitxml test-results/centos7-result/testresults_small_centos7.xml
+
+run_large_tests_centos7:
+	@echo 'Running the super awesome large tests; CentOS 7'
+	pytest -n 2 --reruns 1 -sv tests/test_distributed_splunk_image.py --platform centos-7 --junitxml test-results/centos7-result/testresults_large_centos7.xml
+
+run_small_tests_redhat8:
+	@echo 'Running the super awesome small tests; RedHat 8'
+	pytest -n 3 --reruns 1 -sv tests/test_single_splunk_image.py --platform redhat-8 --junitxml test-results/redhat8-result/testresults_small_redhat8.xml
+
+run_large_tests_redhat8:
+	@echo 'Running the super awesome large tests; RedHat 8'
+	pytest -n 2 --reruns 1 -sv tests/test_distributed_splunk_image.py --platform redhat-8 --junitxml test-results/redhat8-result/testresults_large_redhat8.xml
 
 test_setup:
 	@echo 'Install test requirements'
@@ -316,13 +328,21 @@ test_setup:
 	mkdir test-results/debian10-result || true
 	mkdir test-results/redhat8-result || true
 
-run_tests_debian9:
-	@echo 'Running the super awesome tests; Debian 9'
-	pytest -sv tests/test_docker_splunk.py --platform debian-9 --junitxml test-results/debian9-result/testresults_debian9.xml
+run_small_tests_debian9:
+	@echo 'Running the super awesome small tests; Debian 9'
+	pytest -n 3 --reruns 1 -sv tests/test_single_splunk_image.py --platform debian-9 --junitxml test-results/debian9-result/testresults_small_debian9.xml
+
+run_large_tests_debian9:
+	@echo 'Running the super awesome large tests; Debian 9'
+	pytest -n 2 --reruns 1 -sv tests/test_distributed_splunk_image.py --platform debian-9 --junitxml test-results/debian9-result/testresults_large_debian9.xml
+
+run_small_tests_debian10:
+	@echo 'Running the super awesome small tests; Debian 10'
+	pytest -n 3 --reruns 1 -sv tests/test_single_splunk_image.py --platform debian-10 --junitxml test-results/debian10-result/testresults_small_debian10.xml
 
-run_tests_debian10:
-	@echo 'Running the super awesome tests; Debian 10'
-	pytest -sv tests/test_docker_splunk.py --platform debian-10 --junitxml test-results/debian10-result/testresults_debian10.xml
+run_large_tests_debian10:
+	@echo 'Running the super awesome large tests; Debian 10'
+	pytest -n 2 --reruns 1 -sv tests/test_distributed_splunk_image.py --platform debian-10 --junitxml test-results/debian10-result/testresults_large_debian10.xml
 
 save_containers:
 	@echo 'Saving the following containers:${CONTAINERS_TO_SAVE}'

test_scenarios/1deployment1cm.yaml

@@ -13,7 +13,6 @@ services:
           - appserver
     image: nwang92/nginx-mitm
     hostname: appserver
-    container_name: appserver
     ports:
       - 80
     volumes:
@@ -26,13 +25,13 @@ services:
           - depserver1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: depserver1
-    container_name: depserver1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_ROLE=splunk_deployment_server
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_APPS_URL
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8089
     volumes:
@@ -45,14 +44,14 @@ services:
           - cm1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: cm1
-    container_name: cm1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_DEPLOYMENT_SERVER=depserver1
       - SPLUNK_ROLE=splunk_cluster_master
       - SPLUNK_CLUSTER_MASTER_URL=cm1
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8000
       - 8089

test_scenarios/1deployment1so.yaml

@@ -13,7 +13,6 @@ services:
           - appserver
     image: nwang92/nginx-mitm
     hostname: appserver
-    container_name: appserver
     ports:
       - 80
     volumes:
@@ -26,11 +25,10 @@ services:
           - depserver1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: depserver1
-    container_name: depserver1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_ROLE=splunk_deployment_server
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_APPS_URL
       - DEBUG=true
       - SPLUNK_PASSWORD
     ports:
@@ -43,7 +41,6 @@ services:
           - so1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: so1
-    container_name: so1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_DEPLOYMENT_SERVER=depserver1

test_scenarios/1deployment1uf.yaml

@@ -13,7 +13,6 @@ services:
           - appserver
     image: nwang92/nginx-mitm
     hostname: appserver
-    container_name: appserver
     ports:
       - 80
     volumes:
@@ -26,11 +25,10 @@ services:
           - depserver1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: depserver1
-    container_name: depserver1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_ROLE=splunk_deployment_server
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_APPS_URL
       - DEBUG=true
       - SPLUNK_PASSWORD
     ports:
@@ -43,7 +41,6 @@ services:
           - uf1
     image: ${UF_IMAGE:-splunk/universalforwarder:latest}
     hostname: uf1
-    container_name: uf1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_DEPLOYMENT_SERVER=depserver1

test_scenarios/1hf_splunk_add_user.yaml

@@ -12,7 +12,6 @@ services:
         aliases:
           - hf1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
-    container_name: hf1
     environment:
       - SPLUNK_ROLE=splunk_heavy_forwarder
       - SPLUNK_START_ARGS=--accept-license

test_scenarios/1idx3sh1cm1dep.yaml

@@ -13,7 +13,6 @@ services:
           - appserver
     image: nwang92/nginx-mitm
     hostname: appserver
-    container_name: appserver
     ports:
       - 80
     volumes:
@@ -26,7 +25,6 @@ services:
           - dep1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: dep1
-    container_name: dep1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -38,7 +36,8 @@ services:
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_DEFAULTS_URL
+      - SPLUNK_APPS_URL
     ports:
       - 8000
       - 8089
@@ -52,7 +51,6 @@ services:
           - sh1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: sh1
-    container_name: sh1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -64,6 +62,7 @@ services:
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8000
       - 8089
@@ -77,7 +76,6 @@ services:
           - sh2
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: sh2
-    container_name: sh2
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -89,6 +87,7 @@ services:
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8000
       - 8089
@@ -102,7 +101,6 @@ services:
           - sh3
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: sh3
-    container_name: sh3
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -114,6 +112,7 @@ services:
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8000
       - 8089
@@ -127,7 +126,6 @@ services:
           - cm1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: cm1
-    container_name: cm1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -139,7 +137,8 @@ services:
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_DEFAULTS_URL
+      - SPLUNK_APPS_URL
     ports:
       - 8000
       - 8089
@@ -153,7 +152,6 @@ services:
           - idx1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: idx1
-    container_name: idx1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_INDEXER_URL=idx1
@@ -162,10 +160,11 @@ services:
       - SPLUNK_DEPLOYER_URL=dep1
       - SPLUNK_CLUSTER_MASTER_URL=cm1
       - SPLUNK_ROLE=splunk_indexer
-      - SPLUNK_APPS_URL=http://appserver/splunk_app_example.tgz
+      - SPLUNK_APPS_URL
       - SPLUNK_LICENSE_URI
       - DEBUG=true
       - SPLUNK_PASSWORD
+      - SPLUNK_DEFAULTS_URL
     ports:
       - 8000
       - 8089

test_scenarios/1sh1cm.yaml

@@ -13,7 +13,6 @@ services:
           - sh1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: sh1
-    container_name: sh1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_CLUSTER_MASTER_URL=cm1
@@ -31,7 +30,6 @@ services:
           - cm1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: cm1
-    container_name: cm1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_ROLE=splunk_cluster_master

test_scenarios/1sh1cm1dmc.yaml

@@ -13,7 +13,6 @@ services:
           - sh1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: sh1
-    container_name: sh1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_CLUSTER_MASTER_URL=cm1
@@ -31,7 +30,6 @@ services:
           - cm1
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     hostname: cm1
-    container_name: cm1
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_ROLE=splunk_cluster_master
@@ -49,7 +47,6 @@ services:
     image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
     command: start
     hostname: dmc
-    container_name: dmc
     environment:
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_CLUSTER_MASTER_URL=cm1