test: run UI tests without Saucelabs (#405)

* test: run UI tests without Saucelabs

* ci: use proper path to geckodriver

* ci: actually install poetry

* ci: install pytest-rerunfailures and use correct matrix variable

Author: artemrys

.github/workflows/build-test-release.yaml

@@ -71,11 +71,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: true
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.7"
       - run: |
           curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
       - run: |
@@ -106,17 +104,14 @@ jobs:
             --source=tests/testdata/Splunk_TA_UCCExample/package \
             --config=tests/testdata/Splunk_TA_UCCExample/globalConfig.json \
             --ta-version=0.0.1
-      - uses: actions/upload-artifact@v3
-        with:
-          name: package
-          path: dist/*
         if: always()
       - uses: actions/upload-artifact@v3
         with:
           name: output
           path: output/*
 
   run-ui-tests:
+    name: test-ui Splunk ${{ matrix.splunk.version }} -k ${{ matrix.test_suite }}
     needs:
       - meta
       - build
@@ -145,69 +140,26 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
         with:
-          submodules: recursive
-      - uses: actions/download-artifact@v3
-        with:
-          name: package
-          path: dist/
+          python-version: "3.7"
       - uses: actions/download-artifact@v3
         with:
           name: output
           path: output/
-      - run: |
-          git submodule sync
-          git submodule update --recursive --remote
-      - name: Splunk Testing
+      - run: curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
+      - run: poetry install
+      - name: Link chromedriver and geckodriver
+        # Use installed chromedriver https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
         run: |
-          mkdir test-results
-          export SPLUNK_VERSION=${{ matrix.splunk.version }}
-          SPLUNK_APP_ID=Splunk_TA_UCCExample
-          export SPLUNK_APP_ID
-          SPLUNK_APP_PACKAGE=output/Splunk_TA_UCCExample
-          export SPLUNK_APP_PACKAGE
-          export TEST_SET=tests/ui
-          export TEST_BROWSER="${{ matrix.browser }}"
-          export TEST_SUITE="-k ${{ matrix.test_suite }}"
-          export SAUCE_USERNAME="${{ secrets.SAUCE_USERNAME }}"
-          export SAUCE_PASSWORD="${{ secrets.SAUCE_PASSWORD }}"
-          export SAUCE_TUNNEL_ID="${{ secrets.SAUCE_TUNNEL_ID }}"
-          JOB_NAME="GithubActions::${GITHUB_REPOSITORY#splunk/}[${GITHUB_REF#refs/heads/}][${{ matrix.browser }}] #$(date +%s)"
-          export JOB_NAME
-          SAUCE_IDENTIFIER=$SAUCE_IDENTIFIER-$(cat /proc/sys/kernel/random/uuid)
-          export SAUCE_IDENTIFIER
-          echo "environment setup..."
-          echo "$SPLUNK_VERSION"
-          echo "$SPLUNK_APP_ID"
-          echo "$SPLUNK_APP_PACKAGE"
-          echo "$TEST_SET"
-          echo "$TEST_BROWSER"
-          echo "$TEST_SUITE"
-          echo "Testing..."
-          docker-compose -f docker-compose-ci.yml build
-          # Saucelab connection
-          docker-compose -f docker-compose-ci.yml up -d sauceconnect
-
-          # Start Splunk
-          docker-compose -f docker-compose-ci.yml up -d splunk
-          until docker-compose -f docker-compose-ci.yml logs splunk | grep "Ansible playbook complete" ; do sleep 1; done
-          # Normal Inspecting
-          docker-compose -f docker-compose-ci.yml logs sauceconnect
-          docker ps -a
-          # Run the tests
-          test_exit_code=0
-          docker-compose -f docker-compose-ci.yml up  --abort-on-container-exit test || test_exit_code=$?
-          # Collect Results
-          docker container create --name dummy \
-                              -v addon-factory-smartx-ui-test-library_results:/work/test-results \
-                              registry.access.redhat.com/ubi7/ubi
-          docker cp dummy:/work/test-results/test.xml test-results/ || true
-          docker cp dummy:/work/test-results/test.xml test-results/  || true
-          docker cp dummy:/work/assets test-results/assets  || true
-          docker cp dummy:/work/test-results/report.html test-results/report.html  || true
-          docker-compose -f docker-compose-ci.yml down || true
-          echo exit code "$test_exit_code"
-          exit "$test_exit_code"
+          export PATH=$PATH:$CHROMEWEBDRIVER
+          chromedriver --version
+          export PATH=$PATH:$GECKOWEBDRIVER
+          geckodriver --version
+      - run: |
+          ./run_splunk.sh ${{ matrix.splunk.version }}
+          until curl -Lsk "https://localhost:8088/services/collector/health" &>/dev/null ; do echo -n "Waiting for HEC-" && sleep 5 ; done
+      - run: poetry run pytest tests/ui -k "${{ matrix.test_suite }}" --headless --junitxml=test-results/junit.xml
       - uses: actions/upload-artifact@v3
         if: always()
         with:

Dockerfile-splunk

@@ -52,6 +52,7 @@ Sphinx = "*"
 sphinx_rtd_theme = "*"
 sphinx-panels = "*"
 splunk-add-on-ucc-framework = "^5.28.5"
+pytest-rerunfailures = "^12.0"
 
 [build-system]
 requires = ["poetry_core>=1.0.0"]

Dockerfile-tests

@@ -13,7 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM circleci/python:3.7
-RUN curl https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz -o /home/circleci/saucelabs.tar.gz
-RUN tar -xzf /home/circleci/saucelabs.tar.gz --directory /home/circleci/
-CMD /home/circleci/sc-4.6.2-linux/bin/sc -u $SAUCE_USERNAME -k $SAUCE_PASSWORD -i $SAUCE_TUNNEL_ID --no-remove-colliding-tunnels -v --se-port 4445
+docker run \
+  -v "$PWD/output/Splunk_TA_UCCExample:/opt/splunk/etc/apps/Splunk_TA_UCCExample" \
+  -p 8000:8000 \
+  -p 8088:8088 \
+  -p 8089:8089 \
+  -p 9997:9997 \
+  -e "SPLUNK_START_ARGS=--accept-license" \
+  -e "SPLUNK_PASSWORD=Chang3d!" \
+  -e "SPLUNK_HEC_TOKEN=4a8a737d-5452-426c-a6f7-106dca4e813f" \
+  -e "SPLUNK_DISABLE_POPUPS=true" \
+  -d \
+  --rm \
+  --name splunk splunk/splunk:${1:-latest}

docker-compose-ci.yml

@@ -15,11 +15,14 @@ markers =
     execute_enterprise_cloud_true: Tests to be executed on enterprise cloud
     execute_enterprise_cloud_false: Tests not to be executed on enterprise cloud
 norecursedirs = .git .venv venv build deps tests/deps node_modules package
-addopts = -v -s --tb=long
-    --splunk-type=external
-    --splunk-host=splunk
-    --junitxml=/work/test-results/test.xml
-    --browser=firefox
-    --html=/work/test-results/report.html
-filterwarnings =
-    ignore::DeprecationWarning
+addopts = -vv
+          --splunk-type=external
+          --splunk-host=localhost
+          --splunk-user=admin
+          --splunk-password=Chang3d!
+          --splunk-hec-token=9b741d03-43e9-4164-908b-e09102327d22
+          --browser=chrome
+          --local
+          --persist-browser
+          --log-level=INFO
+          --reruns 3