Add a retry for running tests in nightly and oneMKL interface test actions (#2195)

* Add a retry for running tests in nightly action

* nick-fields/retry action now passes shell arguments

* Add rerun to actions testing oneMKL interfaces

Author: antonwolfy

.github/workflows/check-mkl-interfaces.yaml

@@ -11,6 +11,8 @@ permissions: read-all
 env:
   CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels'
   TEST_ENV_NAME: 'test_onemkl_interfaces'
+  RERUN_TESTS_ON_FAILURE: 'true'
+  RUN_TESTS_MAX_ATTEMPTS: 2
   BUILD_DEP_PKGS: >-
       mkl-devel-dpcpp
       tbb-devel
@@ -95,11 +97,29 @@ jobs:
           mamba install pytest ${{ env.CHANNELS }}
 
       - name: Run tests
+        if: env.RERUN_TESTS_ON_FAILURE != 'true'
         run: |
           python -m pytest -ra --pyargs dpnp.tests
         env:
           SYCL_CACHE_PERSISTENT: 1
 
+      - name: ReRun tests on Linux
+        if: env.RERUN_TESTS_ON_FAILURE == 'true'
+        id: run_tests
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 10
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1
+
   test_by_branch:
     name: Run on ['${{ matrix.os }}', python='${{ matrix.python }}'] with oneMKL develop branch
 
@@ -185,7 +205,25 @@ jobs:
           mamba install pytest ${{ env.CHANNELS }}
 
       - name: Run tests
+        if: env.RERUN_TESTS_ON_FAILURE != 'true'
         run: |
           python -m pytest -ra --pyargs dpnp.tests
         env:
           SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tests on Linux
+        if: env.RERUN_TESTS_ON_FAILURE == 'true'
+        id: run_tests
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 10
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1

.github/workflows/conda-package.yml

@@ -182,14 +182,15 @@ jobs:
         id: run_tests_linux
         uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
         with:
-          shell: bash
           timeout_minutes: 10
           max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
           retry_on: any
           command: |
             . $CONDA/etc/profile.d/conda.sh
-            conda activate ${{ env.TEST_ENV_NAME }}
-            pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
 
   test_windows:
     name: Test ['windows-2019', python='${{ matrix.python }}']
@@ -317,13 +318,11 @@ jobs:
         id: run_tests_win
         uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
         with:
-          shell: cmd
           timeout_minutes: 15
           max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
           retry_on: any
-          command: >-
-            mamba activate ${{ env.TEST_ENV_NAME }}
-            & pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+          command: |
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
 
   upload:
     name: Upload ['${{ matrix.os }}', python='${{ matrix.python }}']

.github/workflows/cron-run-tests.yaml

@@ -15,6 +15,8 @@ env:
   PACKAGE_NAME: dpnp
   CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels'
   TEST_ENV_NAME: test
+  RERUN_TESTS_ON_FAILURE: 'true'
+  RUN_TESTS_MAX_ATTEMPTS: 2
 
 jobs:
   test:
@@ -82,7 +84,38 @@ jobs:
           python -c "import dpnp; print(dpnp.__version__)"
 
       - name: Run tests
+        if: env.RERUN_TESTS_ON_FAILURE != 'true'
         run: |
           python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
         env:
           SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tests on Linux
+        if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner != 'windows-2019'
+        id: run_tests_linux
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 10
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tests on Windows
+        if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner == 'windows-2019'
+        id: run_tests_win
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 15
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1