Enhance Humanitec integration documentation and add new components

- Updated `humanitec-integration.md` to include new configuration and circuit breaker components.
- Introduced `config.py` for integration constants and `circuit_breaker.py` for handling transient failures.
- Added `retryable_http_client.py` to manage HTTP requests with retry logic and circuit breaker protection.
- Updated `humanitec_deployment_deltas.mdx` schema to include new properties: `archived` and `contributors`.
- Enhanced `humanitec_exporter_humanitec_client.mdx` and `humanitec_exporter_port_client.mdx` to utilize the new retryable HTTP client.
- Added methods for bulk and batched entity upserts in the Port client.

Author: lordsarcastic

docs/guides/all/humanitec-integration.md

@@ -12,9 +12,12 @@ import HumanitecResourceBlueprint from "/docs/guides/templates/humanitec/_humani
 import HumanitecResourceGraphBlueprint from "/docs/guides/templates/humanitec/_humanitec_resource_graph_blueprint.mdx";
 import HumanitecExporterCacheScript from "/docs/guides/templates/humanitec/_humanitec_exporter_cache.mdx";
 import HumanitecExporterMainScript from "/docs/guides/templates/humanitec/_humanitec_exporter_main.mdx";
+import HumanitecExporterConfig from "/docs/guides/templates/humanitec/_humanitec_exporter_config.mdx";
 import HumanitecExporterRequirements from "/docs/guides/templates/humanitec/_humanitec_exporter_requirements.mdx";
 import HumanitecExporterPortClient from "/docs/guides/templates/humanitec/_humanitec_exporter_port_client.mdx";
 import HumanitecExporterHumanitecClient from "/docs/guides/templates/humanitec/_humanitec_exporter_humanitec_client.mdx";
+import HumanitecExporterCircuitBreaker from "/docs/guides/templates/humanitec/_humanitec_exporter_circuit_breaker.mdx";
+import HumanitecExporterRetryableHttpClient from "/docs/guides/templates/humanitec/_humanitec_exporter_retryable_http_client.mdx";
 import HumanitecGroups from "/docs/guides/templates/humanitec/_humanitec_groups.mdx";
 import HumanitecUsers from "/docs/guides/templates/humanitec/_humanitec_users.mdx";
 import HumanitecPipelines from "/docs/guides/templates/humanitec/_humanitec_pipelines.mdx";
@@ -103,6 +106,7 @@ In your GitHub repository, [go to **Settings > Secrets**](https://docs.github.co
 
 1. Create the following Python files in a folder named `integration` at the base directory of your GitHub repository:
     - `main.py` - Orchestrates the synchronization of data from Humanitec to Port, ensuring that resource entities are accurately mirrored and updated on your Port catalog.
+    - `config.py` - Contains the configuration constants for the integration, including cache TTL, connection pooling, and other settings.
     - `requirements.txt` - This file contains the dependencies or necessary external packages need to run the integration
 
 
@@ -114,6 +118,13 @@ In your GitHub repository, [go to **Settings > Secrets**](https://docs.github.co
 
     </details>
 
+    <details>
+    <summary><b>Config (Click to expand)</b></summary>
+
+    <HumanitecExporterConfig/>
+
+    </details>
+
 
     <details>
     <summary><b>Requirements (Click to expand)</b></summary>
@@ -127,6 +138,8 @@ In your GitHub repository, [go to **Settings > Secrets**](https://docs.github.co
     - `port_client.py` – Manages authentication and API requests to Port, facilitating the creation and updating of entities within Port's system.
     - `humanitec_client.py` – Handles API interactions with Humanitec, including retrieving data with caching mechanisms to optimize performance.
     - `cache.py` - Provides an in-memory caching mechanism with thread-safe operations for setting, retrieving, and deleting cache entries asynchronously.
+    - `circuit_breaker.py` - Implements a circuit breaker pattern to handle transient failures in API calls, preventing cascading failures and improving the reliability of the integration.
+    - `retryable_http_client.py` - Provides a retryable HTTP client with exponential backoff and jitter to handle failed API calls due to disonnected HTTP connections.
 
   <details>
   <summary><b>Port Client (Click to expand)</b></summary>
@@ -150,6 +163,21 @@ In your GitHub repository, [go to **Settings > Secrets**](https://docs.github.co
 
   </details>
 
+  <details>
+  <summary><b>Circuit Breaker (Click to expand)</b></summary>
+
+  <HumanitecExporterCircuitBreaker/>
+
+  </details>
+
+
+  <details>
+
+  <HumanitecExporterRetryableHttpClient/>
+
+  </details>
+
+
 ### Create the GitHub workflow
 
 Create the file `.github/workflows/humanitec-exporter.yaml` in the `.github/workflows` folder of your repository.

docs/guides/templates/humanitec/_humanitec_deployment_deltas.mdx

@@ -3,56 +3,68 @@
 
 ```json showLineNumbers
 {
-    "identifier": "humanitecDeploymentDelta",
-    "title": "Humanitec Deployment Delta",
-    "icon": "Deployment",
-    "schema": {
-      "properties": {
-        "status": {
-          "title": "Status",
-          "description": "The status of the deployment delta",
-          "type": "string",
-          "icon": "DefaultProperty"
-        },
-        "createdAt": {
-          "title": "Created At",
-          "description": "The date and time when the deployment delta was created",
-          "type": "string",
-          "format": "date-time",
-          "icon": "DefaultProperty"
-        },
-        "createdBy": {
-          "title": "Created By",
-          "description": "The user who created the deployment delta",
-          "type": "string",
-          "icon": "DefaultProperty"
-        },
-        "comment": {
-          "title": "Comment",
-          "description": "Comment for the deployment delta",
-          "type": "string",
-          "icon": "DefaultProperty"
-        },
-        "environment": {
-          "title": "Environment",
-          "description": "The environment for the deployment delta",
-          "type": "string",
-          "icon": "DefaultProperty"
-        }
+  "identifier": "humanitecDeploymentDelta",
+  "title": "Humanitec Deployment Delta",
+  "icon": "Deployment",
+  "schema": {
+    "properties": {
+      "archived": {
+        "title": "Archived",
+        "description": "Whether the deployment delta is archived",
+        "type": "boolean",
+        "icon": "DefaultProperty"
       },
-      "required": []
-    },
-    "mirrorProperties": {},
-    "calculationProperties": {},
-    "aggregationProperties": {},
-    "relations": {
-      "humanitecApplication": {
-        "title": "Application",
-        "target": "humanitecApplication",
-        "required": false,
-        "many": false
+      "contributers": {
+        "title": "Contributers",
+        "description": "The contributers of the deployment delta",
+        "type": "array",
+        "icon": "DefaultProperty"
+      },
+      "createdAt": {
+        "title": "Created At",
+        "description": "The date and time when the deployment delta was created",
+        "type": "string",
+        "format": "date-time",
+        "icon": "DefaultProperty"
+      },
+      "createdBy": {
+        "title": "Created By",
+        "description": "The user who created the deployment delta",
+        "type": "string",
+        "icon": "DefaultProperty"
+      },
+      "modules": {
+        "title": "Modules",
+        "description": "The modules for the deployment delta",
+        "type": "object",
+        "icon": "DefaultProperty"
+      },
+      "shared": {
+        "title": "Shared",
+        "description": "The shared for the deployment delta",
+        "type": "array",
+        "icon": "DefaultProperty"
       }
+    },
+    "required": []
+  },
+  "mirrorProperties": {},
+  "calculationProperties": {},
+  "aggregationProperties": {},
+  "relations": {
+    "humanitecApplication": {
+      "title": "Application",
+      "target": "humanitecApplication",
+      "required": false,
+      "many": false
+    },
+    "humanitecEnvironment": {
+      "title": "Environment",
+      "target": "humanitecEnvironment",
+      "required": false,
+      "many": false
     }
+  }
 }
 ```
 

docs/guides/templates/humanitec/_humanitec_exporter_cache.mdx

@@ -3,6 +3,7 @@
 import asyncio
 from typing import Dict, Any
 
+
 class InMemoryCache:
     def __init__(self):
         self.cache = {}

docs/guides/templates/humanitec/_humanitec_exporter_circuit_breaker.mdx

@@ -0,0 +1,91 @@
+```python showLineNumbers title="circuit_breaker.py"
+
+import time
+import asyncio
+from typing import Optional, Callable, Any
+from loguru import logger
+
+
+class CircuitBreaker:
+    """
+    Circuit breaker pattern to prevent cascading failures.
+    """
+    
+    def __init__(
+        self,
+        failure_threshold: int = 5,
+        recovery_timeout: float = 60.0,
+        expected_exception: type = Exception
+    ):
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.expected_exception = expected_exception
+        
+        self.failure_count = 0
+        self.last_failure_time = None
+        self.state = "CLOSED"  # CLOSED, OPEN, HALF_OPEN
+        
+    def _can_attempt_reset(self) -> bool:
+        """Check if enough time has passed to attempt reset."""
+        if self.last_failure_time is None:
+            return True
+        
+        return time.time() - self.last_failure_time >= self.recovery_timeout
+    
+    def _record_failure(self):
+        """Record a failure and potentially open the circuit."""
+        self.failure_count += 1
+        self.last_failure_time = time.time()
+        
+        if self.failure_count >= self.failure_threshold:
+            self.state = "OPEN"
+            logger.warning(f"Circuit breaker opened after {self.failure_count} failures")
+    
+    def _record_success(self):
+        """Record a success and reset the circuit."""
+        self.failure_count = 0
+        self.last_failure_time = None
+        if self.state == "HALF_OPEN":
+            self.state = "CLOSED"
+            logger.info("Circuit breaker closed after successful request")
+    
+    async def call(self, func: Callable, *args, **kwargs) -> Any:
+        """
+        Execute a function with circuit breaker protection.
+        
+        Args:
+            func: The function to execute
+            *args: Arguments for the function
+            **kwargs: Keyword arguments for the function
+            
+        Returns:
+            The result of the function call
+            
+        Raises:
+            Exception: If the circuit is open or the function fails
+        """
+        if self.state == "OPEN":
+            if self._can_attempt_reset():
+                self.state = "HALF_OPEN"
+                logger.info("Circuit breaker attempting half-open state")
+            else:
+                raise Exception(f"Circuit breaker is OPEN. Last failure: {self.last_failure_time}")
+        
+        try:
+            result = await func(*args, **kwargs)
+            self._record_success()
+            return result
+            
+        except self.expected_exception as e:
+            self._record_failure()
+            raise e
+    
+    def get_state(self) -> str:
+        """Get the current state of the circuit breaker."""
+        return self.state
+    
+    def get_failure_count(self) -> int:
+        """Get the current failure count."""
+        return self.failure_count 
+
+```

docs/guides/templates/humanitec/_humanitec_exporter_config.mdx

@@ -0,0 +1,21 @@
+```python showLineNumbers title="config.py"
+
+"""
+Configuration constants for the Humanitec integration.
+"""
+
+MAX_RETRY_ATTEMPTS = 5
+DEFAULT_TIMEOUT_SECONDS = 30
+RETRY_DELAY_SECONDS = 1
+USE_EXPONENTIAL_BACKOFF = True
+MAX_RETRY_DELAY_SECONDS = 10
+
+MAX_CONNECTIONS = 20
+MAX_KEEPALIVE_CONNECTIONS = 10
+KEEPALIVE_EXPIRY = 30
+
+CACHE_TTL_SECONDS = 300
+
+LOG_LEVEL = "INFO" 
+
+```

docs/guides/templates/humanitec/_humanitec_exporter_humanitec_client.mdx

@@ -1,12 +1,10 @@
 ```python showLineNumbers title="humanitec_client.py"
 
 import httpx
-import asyncio
-from typing import Dict, Any, List
-import datetime
-import re
+from typing import Dict, Any, List, Optional
 from loguru import logger
 from .cache import InMemoryCache
+from .retryable_http_client import RetryableHTTPClient
 
 
 class CACHE_KEYS:
@@ -16,8 +14,13 @@ class CACHE_KEYS:
 
 
 class HumanitecClient:
-    def __init__(self, org_id, api_token, **kwargs) -> None:
-        self.client = kwargs.get("httpx_async_client", httpx.AsyncClient())
+    def __init__(self, org_id: str, api_token: str, **kwargs) -> None:
+        # Inject the retryable HTTP client
+        self.http_client = kwargs.get("http_client")
+        if not self.http_client:
+            timeout = kwargs.get("timeout", httpx.Timeout(20))
+            self.http_client = RetryableHTTPClient(timeout=timeout)
+        
         self.base_url = (
             f"{kwargs.get('base_url','https://api.humanitec.io')}/orgs/{org_id}/"
         )
@@ -36,23 +39,14 @@ class HumanitecClient:
         self,
         method: str,
         endpoint: str,
-        headers: Dict[str, str] | None = None,
-        json: Dict[str, Any] | List[Dict[str, Any]] | None = None,
+        headers: Optional[Dict[str, str]] = None,
+        json: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None,
     ) -> Any:
+        """Send API request using the injected retryable HTTP client."""
         url = self.base_url + endpoint
-        try:
-            logger.debug(f"Requesting Humanitec data for endpoint: {endpoint}")
-            response = await self.client.request(
-                method, url, headers=headers, json=json
-            )
-            response.raise_for_status()
-            return response.json()
-        except httpx.HTTPStatusError as e:
-            logger.error(f"HTTP error occurred: {e.response.text}")
-            raise
-        except Exception as e:
-            logger.error(f"An error occurred: {str(e)}")
-            raise
+        logger.debug(f"Requesting Humanitec data for endpoint: {endpoint}")
+        response = await self.http_client.request(method, url, headers=headers, json=json)
+        return response.json()
 
     async def get_all_applications(self) -> List[Dict[str, Any]]:
         if cached_applications := await self.cache.get(CACHE_KEYS.APPLICATION):
@@ -285,4 +279,10 @@ class HumanitecClient:
         )
         logger.info(f"Received {len(users)} users in group {group_id}")
         return users
+
+    async def close(self):
+        """Close the HTTP client."""
+        if self.http_client:
+            await self.http_client.close()
+
 ```