CLOUDP-325901: Run all test on nightlies (version 2) (#2403)

Signed-off-by: jose.vazquez <jose.vazquez@mongodb.com>

Author: josvazg

.github/codecov.yml

@@ -28,6 +28,13 @@ component_management:
       paths:
         - "!tests/"
         - "!tools/"
+      statuses:    
+        - type: project
+          threshold: -0.5   # Allow -0.5% regression in coverage
+          carryforward: false
+          target: auto
+          informational: true   
+        - type: patch
     - component_id: test-helpers
       name: test-helpers
       paths:

.github/workflows/tests-e2e2.yaml

@@ -74,12 +74,21 @@ jobs:
           E2E2_LABELS: ${{ env.e2e2_labels }}
           USE_JSON: true
         run: |
+          PR_LABELS=${{ env.PR_LABELS }}
+          # Nightly runs all tests, overriding PR labels as '["test/e2e2/*"]'
+          if [ "${{ github.ref }}" == "refs/heads/main" ];then
+            PR_LABELS='["test/e2e2/*"]'
+            echo "Nightly runs all tests"
+          fi
           echo PR_LABELS=${{ env.PR_LABELS }}
           echo E2E2_LABELS=${{ env.E2E2_LABELS }}
-          make compute-labels
-          ./bin/ginkgo-labels > result.json
+          if [ "${PR_LABELS}" == '["test/e2e2/*"]' ]; then
+            echo "["all-e2e2-tests"]" > result.json
+          else
+            make compute-labels
+            ./bin/ginkgo-labels > result.json
+          fi
           echo "E2E2 tests to execute $(cat result.json | jq -c .e2e2)"
-          
           echo "e2e2_matrix=$(cat result.json | jq -c .e2e2)" >> $GITHUB_OUTPUT
 
   compute:
@@ -94,8 +103,10 @@ jobs:
         run: |
           # Note the use of external single quotes to allow for double quotes at inline YAML array
           matrix='["v1.30.10-kind"]'
+          # Nightly runs in both the newest and oldest supported Kubernetes versions
           if [ "${{ github.ref }}" == "refs/heads/main" ];then
             matrix='["v1.30.10-kind", "v1.32.2-kind"]'
+            echo "Nightly runs oldest and newest Kubernetes supported versions"
           fi
           echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
           cat "${GITHUB_OUTPUT}"
@@ -173,7 +184,12 @@ jobs:
           USE_KIND: false
         run: |
           echo "Using ENV: ${{ steps.select-env.outputs.ENV }}"
-          devbox run -- make e2e2 label=${{ env.TEST_NAME }}
+          label=${TEST_NAME}
+          if [ "${TEST_NAME}" == "all-e2e2-tests" ]; then
+            echo "Passing an empty label to run all tests"
+            label=""
+          fi
+          devbox run -- make e2e2 label="${label}"          
 
       - name: Upload operator logs
         if: ${{ failure() }}

test/e2e2/integration_test.go

@@ -56,8 +56,10 @@ var _ = Describe("Atlas Third-Party Integrations Controller", Ordered, Label("in
 	var testNamespace *corev1.Namespace
 
 	_ = BeforeAll(func() {
+		if !version.IsExperimental() {
+			Skip("Skipping experimental test on non experimental build")
+		}
 		deletionProtectionOff := false
-		Expect(version.IsExperimental()).To(BeTrue())
 		ako = runTestAKO(DefaultGlobalCredentials, control.MustEnvVar("OPERATOR_NAMESPACE"), deletionProtectionOff)
 		ako.Start(GinkgoT())
 
@@ -160,62 +162,53 @@ var _ = Describe("Atlas Third-Party Integrations Controller", Ordered, Label("in
 			})
 		},
 		Entry("Test[datadog]: Datadog integration with a parent project",
-			Label("datadog"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/datadog.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/datadog.update.yml")),
 			"atlas-datadog-integ",
 		),
 		Entry("Test[msteams]: Microsoft Teams integration with a parent project",
-			Label("msteams"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/msteams.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/msteams.update.yml")),
 			"atlas-msteams-integ",
 		),
 		Entry("Test[newrelic]: New Relic integration with a parent project",
-			Label("newrelic"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/newrelic.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/newrelic.update.yml")),
 			"atlas-newrelic-integ",
 		),
 		Entry("Test[opsgenie]: Ops Genie integration with a parent project",
-			Label("opsgenie"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/opsgenie.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/opsgenie.update.yml")),
 			"atlas-opsgenie-integ",
 		),
 		Entry("Test[pagerduty]: PagerDuty integration with a parent project",
-			Label("pagerduty"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/pagerduty.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/pagerduty.update.yml")),
 			"atlas-pagerduty-integ",
 		),
 		Entry("Test[prometheus]: Prometheus integration with a parent project",
-			Label("prometheus"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/prometheus.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/prometheus.update.yml")),
 			"atlas-prometheus-integ",
 		),
 		Entry("Test[slack]: Slack integration with a parent project",
-			Label("slack"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/slack.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/slack.update.yml")),
 			"atlas-slack-integ",
 		),
 		Entry("Test[victorops]: Victor Ops integration with a parent project",
-			Label("victorops"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/victorops.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/victorops.update.yml")),
 			"atlas-victorops-integ",
 		),
 		Entry("Test[webhook]: Webhooks integration with a parent project",
-			Label("webhook"),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/webhook.sample.yml")),
 			yml.MustParseObjects(yml.MustOpen(integrations, "integrations/webhook.update.yml")),
 			"atlas-webhook-integ",
 		),
 	)
 
-	It("Can handle isolated integrations", Label("isolated-integration"), func() {
+	It("Can handle isolated integrations", func() {
 		project := akov2.AtlasProject{
 			TypeMeta:   v1.TypeMeta{Kind: "AtlasProject", APIVersion: akov2.GroupVersion.String()},
 			ObjectMeta: v1.ObjectMeta{Name: "atlas-project", Namespace: testNamespace.Name},