cocalc-api/tests: jupyterExecute increase timeout + retry logic

Author: haraldschilly

src/python/cocalc-api/src/cocalc_api/project.py

@@ -10,8 +10,8 @@ def __init__(self, api_key: str, host: str = "https://cocalc.com", project_id: O
         self.project_id = project_id
         self.api_key = api_key
         self.host = host
-        # Use longer timeout for API calls (30 seconds instead of default 5)
-        self.client = httpx.Client(auth=(api_key, ""), headers={"Content-Type": "application/json"}, timeout=30.0)
+        # Use longer timeout for API calls (60 seconds to handle slow kernel startups in CI)
+        self.client = httpx.Client(auth=(api_key, ""), headers={"Content-Type": "application/json"}, timeout=60.0)
 
     def call(self, name: str, arguments: list[Any], timeout: Optional[int] = None) -> Any:
         """
@@ -28,7 +28,7 @@ def call(self, name: str, arguments: list[Any], timeout: Optional[int] = None) -
         payload: dict[str, Any] = {"name": name, "args": arguments, "project_id": self.project_id}
         if timeout is not None:
             payload["timeout"] = timeout
-        resp = self.client.post(self.host + '/api/conat/project', json=payload)
+        resp = self.client.post(self.host + "/api/conat/project", json=payload)
         resp.raise_for_status()
         return handle_error(resp.json())
 

src/python/cocalc-api/tests/test_jupyter.py

@@ -1,6 +1,7 @@
 """
 Tests for Jupyter kernel functionality.
 """
+
 import pytest
 import time
 from typing import Optional
@@ -15,12 +16,12 @@ def test_install_ipykernel(self, project_client):
         result = project_client.system.exec(
             command="python3",
             args=["-m", "pip", "install", "ipykernel"],
-            timeout=120  # 2 minutes should be enough for pip install
+            timeout=120,  # 2 minutes should be enough for pip install
         )
 
         # Check that installation succeeded
-        assert result['exit_code'] == 0
-        assert 'stderr' in result
+        assert result["exit_code"] == 0
+        assert "stderr" in result
 
     def test_install_jupyter_kernel(self, project_client):
         """Test installing the Python 3 Jupyter kernel."""
@@ -33,20 +34,21 @@ def test_install_jupyter_kernel(self, project_client):
                 "install",
                 "--user",  # Install to user location, not system
                 "--name=python3",
-                "--display-name=Python 3"
+                "--display-name=Python 3",
             ],
-            timeout=30)
+            timeout=30,
+        )
 
         # Check that kernel installation succeeded
-        assert result['exit_code'] == 0
+        assert result["exit_code"] == 0
 
 
 class TestJupyterKernels:
     """Tests for Jupyter kernel availability."""
 
     def test_kernels_list_with_project(self, hub, temporary_project):
         """Test getting kernel specs for a specific project."""
-        project_id = temporary_project['project_id']
+        project_id = temporary_project["project_id"]
         kernels = hub.jupyter.kernels(project_id=project_id)
 
         # Should return a list of kernel specs
@@ -55,12 +57,12 @@ def test_kernels_list_with_project(self, hub, temporary_project):
 
     def test_python3_kernel_available(self, hub, temporary_project):
         """Test that the python3 kernel is available after installation."""
-        project_id = temporary_project['project_id']
+        project_id = temporary_project["project_id"]
         kernels = hub.jupyter.kernels(project_id=project_id)
 
         # Extract kernel names from the list
-        kernel_names = [k.get('name') for k in kernels if isinstance(k, dict)]
-        assert 'python3' in kernel_names
+        kernel_names = [k.get("name") for k in kernels if isinstance(k, dict)]
+        assert "python3" in kernel_names
 
 
 class TestJupyterExecuteViaHub:
@@ -69,64 +71,64 @@ class TestJupyterExecuteViaHub:
     @pytest.mark.skip(reason="hub.jupyter.execute() has timeout issues - use project.system.jupyter_execute() instead")
     def test_execute_simple_sum(self, hub, temporary_project):
         """Test executing a simple sum using the python3 kernel."""
-        project_id = temporary_project['project_id']
+        project_id = temporary_project["project_id"]
 
-        result = hub.jupyter.execute(input='sum(range(100))', kernel='python3', project_id=project_id)
+        result = hub.jupyter.execute(input="sum(range(100))", kernel="python3", project_id=project_id)
 
         # Check the result structure
         assert isinstance(result, dict)
-        assert 'output' in result
+        assert "output" in result
 
         # Check that we got the correct result (sum of 0..99 = 4950)
-        output = result['output']
+        output = result["output"]
         assert len(output) > 0
 
         # Extract the result from the output
         # Format: [{'data': {'text/plain': '4950'}}]
         first_output = output[0]
-        assert 'data' in first_output
-        assert 'text/plain' in first_output['data']
-        assert first_output['data']['text/plain'] == '4950'
+        assert "data" in first_output
+        assert "text/plain" in first_output["data"]
+        assert first_output["data"]["text/plain"] == "4950"
 
     @pytest.mark.skip(reason="hub.jupyter.execute() has timeout issues - use project.system.jupyter_execute() instead")
     def test_execute_with_history(self, hub, temporary_project):
         """Test executing code with history context."""
-        project_id = temporary_project['project_id']
+        project_id = temporary_project["project_id"]
 
-        result = hub.jupyter.execute(history=['a = 100'], input='sum(range(a + 1))', kernel='python3', project_id=project_id)
+        result = hub.jupyter.execute(history=["a = 100"], input="sum(range(a + 1))", kernel="python3", project_id=project_id)
 
         # Check the result (sum of 0..100 = 5050)
         assert isinstance(result, dict)
-        assert 'output' in result
+        assert "output" in result
 
-        output = result['output']
+        output = result["output"]
         assert len(output) > 0
 
         first_output = output[0]
-        assert 'data' in first_output
-        assert 'text/plain' in first_output['data']
-        assert first_output['data']['text/plain'] == '5050'
+        assert "data" in first_output
+        assert "text/plain" in first_output["data"]
+        assert first_output["data"]["text/plain"] == "5050"
 
     @pytest.mark.skip(reason="hub.jupyter.execute() has timeout issues - use project.system.jupyter_execute() instead")
     def test_execute_print_statement(self, hub, temporary_project):
         """Test executing code that prints output."""
-        project_id = temporary_project['project_id']
+        project_id = temporary_project["project_id"]
 
-        result = hub.jupyter.execute(input='print("Hello from Jupyter")', kernel='python3', project_id=project_id)
+        result = hub.jupyter.execute(input='print("Hello from Jupyter")', kernel="python3", project_id=project_id)
 
         # Check that we got output
         assert isinstance(result, dict)
-        assert 'output' in result
+        assert "output" in result
 
-        output = result['output']
+        output = result["output"]
         assert len(output) > 0
 
         # Print statements produce stream output
         first_output = output[0]
-        assert 'name' in first_output
-        assert first_output['name'] == 'stdout'
-        assert 'text' in first_output
-        assert 'Hello from Jupyter' in first_output['text']
+        assert "name" in first_output
+        assert first_output["name"] == "stdout"
+        assert "text" in first_output
+        assert "Hello from Jupyter" in first_output["text"]
 
 
 class TestJupyterExecuteViaProject:
@@ -147,10 +149,10 @@ def test_jupyter_execute_simple_sum(self, project_client):
 
         for attempt in range(max_retries):
             try:
-                result = project_client.system.jupyter_execute(input='sum(range(100))', kernel='python3')
+                result = project_client.system.jupyter_execute(input="sum(range(100))", kernel="python3")
                 break
             except RuntimeError as e:
-                if 'timeout' in str(e).lower() and attempt < max_retries - 1:
+                if "timeout" in str(e).lower() and attempt < max_retries - 1:
                     print(f"Attempt {attempt + 1} timed out, retrying in {retry_delay}s...")
                     time.sleep(retry_delay)
                 else:
@@ -162,45 +164,59 @@ def test_jupyter_execute_simple_sum(self, project_client):
 
         # Check that we got the correct result (sum of 0..99 = 4950)
         first_output = result[0]
-        assert 'data' in first_output
-        assert 'text/plain' in first_output['data']
-        assert first_output['data']['text/plain'] == '4950'
+        assert "data" in first_output
+        assert "text/plain" in first_output["data"]
+        assert first_output["data"]["text/plain"] == "4950"
 
     def test_jupyter_execute_with_history(self, project_client):
         """
         Test executing code with history via project API.
 
         The result is a list of output items directly.
         """
-        result = project_client.system.jupyter_execute(history=['b = 50'], input='b * 2', kernel='python3')
+        result = project_client.system.jupyter_execute(history=["b = 50"], input="b * 2", kernel="python3")
 
         # Result is a list
         assert isinstance(result, list)
         assert len(result) > 0
 
         # Check the result (50 * 2 = 100)
         first_output = result[0]
-        assert 'data' in first_output
-        assert 'text/plain' in first_output['data']
-        assert first_output['data']['text/plain'] == '100'
+        assert "data" in first_output
+        assert "text/plain" in first_output["data"]
+        assert first_output["data"]["text/plain"] == "100"
 
     def test_jupyter_execute_list_operation(self, project_client):
         """
         Test executing code that works with lists.
 
         The result is a list of output items directly.
         """
-        result = project_client.system.jupyter_execute(input='[x**2 for x in range(5)]', kernel='python3')
+        # Retry logic for kernel startup
+        max_retries = 3
+        retry_delay = 15
+        result: Optional[list] = None
+
+        for attempt in range(max_retries):
+            try:
+                result = project_client.system.jupyter_execute(input="[x**2 for x in range(5)]", kernel="python3")
+                break
+            except RuntimeError as e:
+                if "timeout" in str(e).lower() and attempt < max_retries - 1:
+                    print(f"Attempt {attempt + 1} timed out, retrying in {retry_delay}s...")
+                    time.sleep(retry_delay)
+                else:
+                    raise
 
         # Result is a list
         assert isinstance(result, list)
         assert len(result) > 0
 
         # Check the result ([0, 1, 4, 9, 16])
         first_output = result[0]
-        assert 'data' in first_output
-        assert 'text/plain' in first_output['data']
-        assert first_output['data']['text/plain'] == '[0, 1, 4, 9, 16]'
+        assert "data" in first_output
+        assert "text/plain" in first_output["data"]
+        assert first_output["data"]["text/plain"] == "[0, 1, 4, 9, 16]"
 
 
 class TestJupyterKernelManagement:
@@ -209,7 +225,20 @@ class TestJupyterKernelManagement:
     def test_list_jupyter_kernels(self, project_client):
         """Test listing running Jupyter kernels."""
         # First execute some code to ensure a kernel is running
-        project_client.system.jupyter_execute(input='1+1', kernel='python3')
+        # Retry logic for first kernel startup (may take longer in CI)
+        max_retries = 3
+        retry_delay = 15
+
+        for attempt in range(max_retries):
+            try:
+                project_client.system.jupyter_execute(input="1+1", kernel="python3")
+                break
+            except RuntimeError as e:
+                if "timeout" in str(e).lower() and attempt < max_retries - 1:
+                    print(f"Attempt {attempt + 1} timed out, retrying in {retry_delay}s...")
+                    time.sleep(retry_delay)
+                else:
+                    raise
 
         # List kernels
         kernels = project_client.system.list_jupyter_kernels()
@@ -222,30 +251,30 @@ def test_list_jupyter_kernels(self, project_client):
 
         # Each kernel should have required fields
         for kernel in kernels:
-            assert 'pid' in kernel
-            assert 'connectionFile' in kernel
-            assert isinstance(kernel['pid'], int)
-            assert isinstance(kernel['connectionFile'], str)
+            assert "pid" in kernel
+            assert "connectionFile" in kernel
+            assert isinstance(kernel["pid"], int)
+            assert isinstance(kernel["connectionFile"], str)
 
     def test_stop_jupyter_kernel(self, project_client):
         """Test stopping a specific Jupyter kernel."""
         # Execute code to start a kernel
-        project_client.system.jupyter_execute(input='1+1', kernel='python3')
+        project_client.system.jupyter_execute(input="1+1", kernel="python3")
 
         # List kernels
         kernels = project_client.system.list_jupyter_kernels()
         assert len(kernels) > 0
 
         # Stop the first kernel
         kernel_to_stop = kernels[0]
-        result = project_client.system.stop_jupyter_kernel(pid=kernel_to_stop['pid'])
+        result = project_client.system.stop_jupyter_kernel(pid=kernel_to_stop["pid"])
 
         # Should return success
         assert isinstance(result, dict)
-        assert 'success' in result
-        assert result['success'] is True
+        assert "success" in result
+        assert result["success"] is True
 
         # Verify kernel is no longer in the list
         kernels_after = project_client.system.list_jupyter_kernels()
-        remaining_pids = [k['pid'] for k in kernels_after]
-        assert kernel_to_stop['pid'] not in remaining_pids
+        remaining_pids = [k["pid"] for k in kernels_after]
+        assert kernel_to_stop["pid"] not in remaining_pids