refactor: test and automation scripts

.github/workflows/ci.yml

@@ -5,6 +5,13 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run with upterm debugging enabled (requires SSH key in secrets)'
+        required: false
+        default: false
 
 env:
   HELM_VERSION: v3.15.2
@@ -27,7 +34,7 @@ jobs:
           version: ${{ env.HELM_VERSION }}
 
       - name: Setup Helm dependencies
-        run: ./scripts/deploy.sh setup
+        run: ./scripts/deploy.sh setup --deps-only
 
       - name: Install ajv-cli
         run: npm install -g ajv-cli ajv-formats
@@ -39,7 +46,7 @@ jobs:
         run: make validate-schema
 
       - name: Run Helm unit tests
-        run: make tests
+        run: make test-helm
 
   integration-tests:
     name: Integration tests
@@ -49,6 +56,16 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install httpx psycopg2-binary pytest
+
       - name: Start K3s cluster
         uses: jupyterhub/action-k3s-helm@v4
         with:
@@ -62,48 +79,83 @@ jobs:
 
       - name: Wait for K3s readiness
         run: |
-          echo "=== Waiting for K3s cluster to be ready ==="
-
-          # The action already sets up kubectl context, just verify it works
-          kubectl cluster-info
-          kubectl get nodes
-
-          # Wait for core components
-          kubectl wait --for=condition=Ready pod -l k8s-app=kube-dns -n kube-system --timeout=300s
-          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=traefik -n kube-system --timeout=300s
-
-          # Verify Traefik CRDs
-          timeout=300; counter=0
-          for crd in "middlewares.traefik.io" "ingressroutes.traefik.io"; do
-            while [ $counter -lt $timeout ] && ! kubectl get crd "$crd" &>/dev/null; do
-              sleep 3; counter=$((counter + 3))
-            done
-            [ $counter -ge $timeout ] && { echo "❌ Timeout waiting for $crd"; exit 1; }
-          done
-
-          echo "✅ K3s cluster ready"
+          # Source validation library and use K3s readiness check
+          source ./scripts/lib/common.sh
+          source ./scripts/lib/validation.sh
+          validate_k3s_readiness
 
       - name: Deploy eoAPI
         id: deploy
         run: |
+          set -e  # Exit on any error
+
           echo "=== eoAPI Deployment ==="
+          echo "RELEASE_NAME: ${RELEASE_NAME}"
+          echo "PGO_VERSION: ${{ env.PGO_VERSION }}"
+
           export RELEASE_NAME="${RELEASE_NAME}"
           export PGO_VERSION="${{ env.PGO_VERSION }}"
-          export CI_MODE=true
 
-          # Deploy using consolidated script with CI mode
-          ./scripts/deploy.sh --ci
+          # Deploy using consolidated script with k3s values for testing
+          echo "Running deploy script..."
+          if ! ./scripts/deploy.sh deploy --namespace "${RELEASE_NAME}" --release "${RELEASE_NAME}" \
+            -f charts/eoapi/local-base-values.yaml \
+            -f charts/eoapi/local-k3s-values.yaml \
+            --set ingress.host=eoapi.local \
+            --set eoapi-notifier.config.sources[0].config.connection.existingSecret.name="${RELEASE_NAME}-pguser-eoapi" \
+            --debug; then
+
+            echo "❌ Deploy script failed"
+            kubectl get namespaces
+            kubectl get pods -A
+            exit 1
+          fi
+
+          # Run post-deployment validation
+          echo "=== Running post-deployment validation ==="
+          ./scripts/deploy.sh validate \
+            --namespace "${RELEASE_NAME}" \
+            --release "${RELEASE_NAME}" \
+            --verbose
+
+      - name: Debug session after deployment
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+        uses: lhotari/action-upterm@v1
+        with:
+          limit-access-to-keys: ${{ secrets.UPTERM_SSH_KEY }}
+          wait-timeout-minutes: 30
 
-      - name: Validate deployment
+      - name: Run deployment tests
         run: |
-          echo "=== Post-deployment validation ==="
+          echo "=== Running deployment tests ==="
+          export NAMESPACE="${RELEASE_NAME}"
           ./scripts/test.sh check-deployment
 
+      - name: Wait for services to be ready
+        run: |
+          # Use deploy.sh validate command to check service readiness and API connectivity
+          export INGRESS_HOST="eoapi.local"
+          ./scripts/deploy.sh validate \
+            --namespace "${RELEASE_NAME}" \
+            --release "${RELEASE_NAME}" \
+            --debug
+
       - name: Run integration tests
         run: |
           export RELEASE_NAME="$RELEASE_NAME"
+          export NAMESPACE="$RELEASE_NAME"
+          export STAC_ENDPOINT="http://eoapi.local/stac"
+          export RASTER_ENDPOINT="http://eoapi.local/raster"
+          export VECTOR_ENDPOINT="http://eoapi.local/vector"
           ./scripts/test.sh integration --debug
 
+      - name: Debug session on test failure
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
+        uses: lhotari/action-upterm@v1
+        with:
+          limit-access-to-keys: ${{ secrets.UPTERM_SSH_KEY }}
+          wait-timeout-minutes: 30
+
       - name: Debug failed deployment
         if: failure()
         run: |
@@ -112,8 +164,8 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
-          helm uninstall "$RELEASE_NAME" -n eoapi || true
-          kubectl delete namespace eoapi || true
+          helm uninstall "$RELEASE_NAME" -n "$RELEASE_NAME" || true
+          kubectl delete namespace "$RELEASE_NAME" || true
   validate-docs:
     name: Validate documentation
     runs-on: ubuntu-latest

.github/workflows/tests/test_notifications.py

@@ -1,6 +1,7 @@
 """Test notification system deployment and functionality."""
 
 import json
+import os
 import subprocess
 import time
 
@@ -9,6 +10,9 @@
 
 def test_eoapi_notifier_deployment():
     """Test that eoapi-notifier deployment is running."""
+    # Get namespace from environment variable
+    namespace = os.environ.get("NAMESPACE", "eoapi")
+
     # Check if eoapi-notifier deployment exists and is ready
     result = subprocess.run(
         [
@@ -18,7 +22,7 @@ def test_eoapi_notifier_deployment():
             "-l",
             "app.kubernetes.io/name=eoapi-notifier",
             "-n",
-            "eoapi",
+            namespace,
             "--no-headers",
             "-o",
             "custom-columns=READY:.status.readyReplicas",
@@ -40,6 +44,9 @@ def test_eoapi_notifier_deployment():
 
 def test_cloudevents_sink_exists():
     """Test that Knative CloudEvents sink service exists and is accessible."""
+    # Get namespace from environment variable
+    namespace = os.environ.get("NAMESPACE", "eoapi")
+
     # Check if Knative service exists
     result = subprocess.run(
         [
@@ -48,6 +55,8 @@ def test_cloudevents_sink_exists():
             "ksvc",
             "-l",
             "app.kubernetes.io/component=cloudevents-sink",
+            "-n",
+            namespace,
             "--no-headers",
         ],
         capture_output=True,
@@ -66,6 +75,9 @@ def test_cloudevents_sink_exists():
 
 def test_notification_configuration():
     """Test that eoapi-notifier is configured correctly."""
+    # Get namespace from environment variable
+    namespace = os.environ.get("NAMESPACE", "eoapi")
+
     # Get the configmap for eoapi-notifier
     result = subprocess.run(
         [
@@ -74,6 +86,8 @@ def test_notification_configuration():
             "configmap",
             "-l",
             "app.kubernetes.io/name=eoapi-notifier",
+            "-n",
+            namespace,
             "-o",
             r"jsonpath={.items[0].data.config\.yaml}",
         ],
@@ -96,16 +110,19 @@ def test_notification_configuration():
 
 def test_cloudevents_sink_logs_show_startup():
     """Test that Knative CloudEvents sink started successfully."""
+    # Get namespace from environment variable
+    namespace = os.environ.get("NAMESPACE", "eoapi")
+
     # Get Knative CloudEvents sink pod logs
     result = subprocess.run(
         [
             "kubectl",
             "logs",
             "-l",
-            "serving.knative.dev/service",
+            "serving.knative.dev/service=eoapi-cloudevents-sink",
             "-n",
-            "eoapi",
-            "--tail=20",
+            namespace,
+            "--tail=100",
         ],
         capture_output=True,
         text=True,
@@ -115,7 +132,10 @@ def test_cloudevents_sink_logs_show_startup():
         pytest.skip("Cannot get Knative CloudEvents sink logs")
 
     logs = result.stdout
-    assert "listening on port" in logs, "Knative CloudEvents sink should have started successfully"
+    # Check for either startup message or evidence the service is running
+    assert "listening on port" in logs or "received a request" in logs, (
+        f"Knative CloudEvents sink should be running. Got logs: {logs[:500]}"
+    )
 
 
 def test_eoapi_notifier_logs_show_connection():