Reduce number of test cases (cvat-ai#24)

Author: sizov-kirill

.github/workflows/main.yml

@@ -16,9 +16,11 @@ jobs:
       runs-on: ubuntu-latest
       steps:
       - uses: actions/checkout@v2
+
       - uses: actions/setup-python@v2
         with:
           python-version: '3.8'
+
       - name: Getting SHA from the default branch
         id: get-sha
         run: |
@@ -36,6 +38,7 @@ jobs:
 
           echo ::set-output name=default_branch::${DEFAULT_BRANCH}
           echo ::set-output name=sha::${SHA}
+
       - name: Waiting a cache creation in the default branch
         if: ${{ github.ref_name != 'develop' }}
         run: |
@@ -69,13 +72,22 @@ jobs:
             echo "Number of attempts expired!"
             echo "Probably the creation of the cache is not yet complete. Will continue working without the cache."
           fi
+
       - name: Getting CVAT server cache from the default branch
         uses: actions/cache@v2
         with:
           path: /tmp/cvat_cache_server
           key: ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
+
+      - name: Getting CVAT UI cache from the default branch
+        uses: actions/cache@v2
+        with:
+          path: /tmp/cvat_cache_ui
+          key: ${{ runner.os }}-build-ui-${{ steps.get-sha.outputs.sha }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1.1.2
+
       - name: Building CVAT server image
         uses: docker/build-push-action@v2
         with:
@@ -84,11 +96,22 @@ jobs:
           cache-from: type=local,src=/tmp/cvat_cache_server
           tags: openvino/cvat_server:latest
           load: true
+
+      - name: Building CVAT UI image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile.ui
+          cache-from: type=local,src=/tmp/cvat_cache_ui
+          tags: openvino/cvat_ui:latest
+          load: true
+
       - name: Running OPA tests
         run: |
           curl -L -o opa https://openpolicyagent.org/downloads/v0.34.2/opa_linux_amd64_static
           chmod +x ./opa
           ./opa test cvat/apps/iam/rules
+
       - name: Running REST API tests
         env:
           API_ABOUT_PAGE: "localhost:8080/api/server/about"
@@ -98,16 +121,22 @@ jobs:
         run: |
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml up -d
           /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
+
           pip3 install --user -r tests/rest_api/requirements.txt
-          pytest tests/rest_api/
+          pytest tests/rest_api/ -k 'GET'
+
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml down -v
+
       - name: Running unit tests
         env:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
+          -c 'coverage run -a manage.py test cvat/apps utils/cli -k 'tasks_id' -k 'lambda' -k 'share' && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
+          -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
       - name: Uploading code coverage results as an artifact
         if: github.ref == 'refs/heads/develop'
         uses: actions/upload-artifact@v2
@@ -126,9 +155,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_objects', 'actions_objects2', 'actions_users', 'actions_projects_models', 'actions_organizations', 'canvas3d_functionality', 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2']
+        specs: ['canvas3d_functionality', 'actions']
     steps:
       - uses: actions/checkout@v2
+
       - name: Getting SHA from the default branch
         id: get-sha
         run: |
@@ -146,6 +176,7 @@ jobs:
 
           echo ::set-output name=default_branch::${DEFAULT_BRANCH}
           echo ::set-output name=sha::${SHA}
+
       - name: Waiting a cache creation in the default branch
         run: |
           URL_runs="https://api.github.com/repos/${{ github.repository }}/actions/workflows/cache.yml/runs"
@@ -179,21 +210,26 @@ jobs:
             echo "Number of attempts expired!"
             echo "Probably the creation of the cache is not yet complete. Will continue working without the cache."
           fi
+
       - name: Getting CVAT server cache from the default branch
         uses: actions/cache@v2
         with:
           path: /tmp/cvat_cache_server
           key: ${{ runner.os }}-build-server-${{ steps.get-sha.outputs.sha }}
-      - name: Getting cache CVAT UI from the default branch
+
+      - name: Getting CVAT UI cache from the default branch
         uses: actions/cache@v2
         with:
           path: /tmp/cvat_cache_ui
           key: ${{ runner.os }}-build-ui-${{ steps.get-sha.outputs.sha }}
+
       - uses: actions/setup-node@v2
         with:
             node-version: '16.x'
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1.1.2
+
       - name: Building CVAT server image
         uses: docker/build-push-action@v2
         with:
@@ -202,6 +238,7 @@ jobs:
           cache-from: type=local,src=/tmp/cvat_cache_server
           tags: openvino/cvat_server:latest
           load: true
+
       - name: Building CVAT UI image
         uses: docker/build-push-action@v2
         with:
@@ -210,12 +247,14 @@ jobs:
           cache-from: type=local,src=/tmp/cvat_cache_ui
           tags: openvino/cvat_ui:latest
           load: true
+
       - name: Instrumentation of the code then rebuilding the CVAT UI
         if: github.ref == 'refs/heads/develop'
         run: |
           npm ci
           npm run coverage
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml build cvat_ui
+
       - name: Running e2e tests
         env:
           DJANGO_SU_NAME: 'admin'
@@ -226,38 +265,51 @@ jobs:
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f tests/docker-compose.file_share.yml up -d
           /bin/bash -c 'while [[ $(curl -s -o /dev/null -w "%{http_code}" ${API_ABOUT_PAGE}) != "401" ]]; do sleep 5; done'
           docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+
           cd ./tests
           npm ci
-          if [[ ${{ github.ref }} == 'refs/heads/develop' ]]; then
-            if [ ${{ matrix.specs }} == 'canvas3d_functionality' ] || [ ${{ matrix.specs }} == 'canvas3d_functionality_2' ]; then
-              npx cypress run --headed --browser chrome --config-file cypress_canvas3d.json --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
-            else
-              npx cypress run --browser chrome --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
-            fi
-            mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
+
+          if [ ${{ matrix.specs }} == 'canvas3d_functionality' ]; then
+            # Choosing 5 test files
+            selected_files=$(find ./cypress/integration |  grep -e 'case.*\|issue.*' | grep js | grep 3d | sort | head -5 | tr '\n' ',')
+
+            npx cypress run \
+              --headed \
+              --browser chrome \
+              --env coverage=false \
+              --config-file cypress_canvas3d.json \
+              --spec "${selected_files} cypress/integration/remove_users_tasks_projects_organizations.js"
           else
-            if [ ${{ matrix.specs }} == 'canvas3d_functionality' ] || [ ${{ matrix.specs }} == 'canvas3d_functionality_2' ]; then
-              npx cypress run --headed --browser chrome --env coverage=false --config-file cypress_canvas3d.json --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
-            else
-              npx cypress run --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
-            fi
+            # Choosing 20 test files
+            find ./cypress/integration |  grep -e 'case.*\|issue.*' | grep js | sed '/.*3d.*/d' | sort > test_files
+            selected_files=$({ head -10; tail -10;} < test_files | tr '\n' ',')
+            rm test_files
+
+            npx cypress run \
+              --browser chrome \
+              --env coverage=false \
+              --spec "${selected_files} cypress/integration/remove_users_tasks_projects_organizations.js"
           fi
+
       - name: Creating a log file from "cvat" container logs
         if: failure()
         run: |
             docker logs cvat > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
+
       - name: Uploading cypress screenshots as an artifact
         if: failure()
         uses: actions/upload-artifact@v2
         with:
           name: cypress_screenshots_${{ matrix.specs }}
           path: ${{ github.workspace }}/tests/cypress/screenshots
+
       - name: Uploading "cvat" container logs as an artifact
         if: failure()
         uses: actions/upload-artifact@v2
         with:
           name: cvat_container_logs
           path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
+
       - name: Uploading code coverage results as an artifact
         if: github.ref == 'refs/heads/develop'
         uses: actions/upload-artifact@v2

tests/rest_api/conftest.py

@@ -35,7 +35,7 @@ def init_test_db():
     _run('docker exec cvat_db psql -U root -d postgres -v from=test_db -v to=cvat -f restore.sql')
     _run('docker exec cvat_db dropdb test_db')
 
-@pytest.fixture(scope='function', autouse=True)
+@pytest.fixture(scope='function')
 def restore():
     _run('docker exec cvat_db psql -U root -d postgres -v from=test_db -v to=cvat -f restore.sql')
 

tests/rest_api/test_cloud_storages.py

@@ -59,6 +59,7 @@ def test_org_user_get_coud_storage(self, org_id, storage_id, role, is_owner, is_
             self._test_cannot_see(username, storage_id, org_id=org_id)
 
 
+@pytest.mark.usefixtures("restore")
 class TestPostCloudStorage:
     _SPEC = {
         'provider_type': 'AWS_S3_BUCKET',
@@ -120,6 +121,7 @@ def test_org_user_create_coud_storage(self, org_id, role, is_allow, find_users):
         else:
             self._test_cannot_create(username, self._SPEC, org_id=org_id)
 
+@pytest.mark.usefixtures("restore")
 class TestPatchCloudStorage:
     _SPEC = {
         'display_name': 'New display name',

tests/rest_api/test_invitations.py

@@ -6,6 +6,7 @@
 import pytest
 from .utils.config import post_method
 
+@pytest.mark.usefixtures("restore")
 class TestCreateInvitations:
     def _test_post_invitation_201(self, user, data, invitee, **kwargs):
         response = post_method(user, 'invitations', data, **kwargs)

tests/rest_api/test_issues.py

@@ -9,6 +9,7 @@
 
 from .utils.config import post_method, patch_method
 
+@pytest.mark.usefixtures("restore")
 class TestPostIssues:
     def _test_check_response(self, user, data, is_allow, **kwargs):
         response = post_method(user, 'issues', data, **kwargs)
@@ -79,6 +80,7 @@ def test_member_create_issue(self, org, role, job_staff, is_allow,
         self._test_check_response(username, data, is_allow, org_id=org)
 
 
+@pytest.mark.usefixtures("restore")
 class TestPatchIssues:
     def _test_check_response(self, user, issue_id, data, is_allow, **kwargs):
         response = patch_method(user, f'issues/{issue_id}', data,

tests/rest_api/test_jobs.py

@@ -180,7 +180,7 @@ def test_non_member_get_job_annotations(self, org, privilege, is_allow,
                 job_id, annotations['job'][str(job_id)], **kwargs)
         else:
             self._test_get_job_annotations_403(username, job_id, **kwargs)
-
+@pytest.mark.usefixtures("restore")
 class TestPatchJobAnnotations:
     _ORG = 2
 
@@ -259,6 +259,7 @@ def test_user_update_job_annotations(self, org, privilege, job_staff, is_allow,
 
         self._test_check_respone(is_allow, response, data)
 
+@pytest.mark.usefixtures("restore")
 class TestPatchJob:
     _ORG = 2
 

tests/rest_api/test_memberships.py

@@ -42,6 +42,7 @@ def test_non_members_cannot_see_members_membership(self):
             self._test_cannot_see_memberships(user, org_id=1)
 
 
+@pytest.mark.usefixtures("restore")
 class TestPatchMemberships:
     _ORG = 2
 

tests/rest_api/test_organizations.py

@@ -60,6 +60,7 @@ def test_can_see_specific_organization(self, privilege, role, is_member,
         else:
             assert response.status_code == HTTPStatus.NOT_FOUND
 
+@pytest.mark.usefixtures("restore")
 class TestPatchOrganizations:
     _ORG = 2
 
@@ -100,6 +101,7 @@ def test_can_update_specific_organization(self, privilege, role, is_member,
         else:
             assert response.status_code != HTTPStatus.OK
 
+@pytest.mark.usefixtures("restore")
 class TestDeleteOrganizations:
     _ORG = 2
 

tests/rest_api/test_projects.py

@@ -110,6 +110,7 @@ def test_if_org_member_supervisor_or_worker_can_see_project(self, projects,
 
         self._test_response_200(user_in_project['username'], project_id, org_id=user_in_project['org'])
 
+@pytest.mark.usefixtures("restore")
 class TestPostProjects:
     def _test_create_project_201(self, user, spec, **kwargs):
         response = post_method(user, '/projects', spec, **kwargs)

tests/rest_api/test_remote_url.py

@@ -2,8 +2,11 @@
 #
 # SPDX-License-Identifier: MIT
 
-from time import sleep
 from http import HTTPStatus
+from time import sleep
+
+import pytest
+
 from .utils.config import get_method, post_method
 
 
@@ -26,6 +29,7 @@ def _wait_until_task_is_created(username, task_id):
         sleep(1)
 
 
+@pytest.mark.usefixtures("restore")
 class TestGetAnalytics:
     task_id = 12
     def _test_can_create(self, user, task_id, resources):