RHAIENG-1161: test(dockerfile): add test for probe logic and notebook args in codeserver (opendatahub-io#2504)

* RHAIENG-1161: test(dockerfile): add test for probe logic and notebook args in codeserver

This way it should now be impossible to build a container image that is then unable to start.

```
nginx: [alert] could not open error log file: open() "/var/log/nginx/error.log" failed (13: Permission denied)
```

* script -qc "/opt/app-root/bin/run-code-server.sh" /dev/null & works

Author: jiridanek

Makefile

@@ -526,4 +526,4 @@ print-release:
 .PHONY: test
 test:
 	@echo "Running quick static tests"
-	uv run pytest
+	uv run pytest -m 'not buildonlytest'

codeserver/ubi9-python-3.12/Dockerfile.cpu

@@ -270,3 +270,15 @@ RUN chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \
 WORKDIR /opt/app-root/src
 
 CMD ["/opt/app-root/bin/run-code-server.sh"]
+
+FROM codeserver as tests
+ARG CODESERVER_SOURCE_CODE=codeserver/ubi9-python-3.12
+COPY ${CODESERVER_SOURCE_CODE}/test /tmp/test
+# TODO(jdanek): add --mount=type=bind,target=/opt/app-root/src
+RUN <<'EOF'
+set -Eeuxo pipefail
+python3 /tmp/test/test_startup.py |& tee /tmp/test_log.txt
+EOF
+
+from codeserver
+COPY --from=tests /tmp/test_log.txt /tmp/test_log.txt

codeserver/ubi9-python-3.12/test/README.md

@@ -0,0 +1,39 @@
+## test_codeserver_startup
+
+Checks that the image entrypoint starts up and the image passes the readiness probe.
+
+### Konflux considerations
+
+Workaround for [KFLUXSPRT-5139](https://redhat-internal.slack.com/archives/C04PZ7H0VA8/p1758128206734419):
+
+Rootless (or more like remote?) buildah, or is it shipwright on openshift, has `/dev/stderr` be a pipe that may not be reopened.
+Since startup script creates some symlinks that then lead nginx to attempt to reopen `/dev/stderr`, it causes the following error:
+
+    nginx: [alert] could not open error log file: open() "/var/log/nginx/error.log" failed (13: Permission denied)
+
+Here's how it looks like in the Konflux container:
+
+```
+(app-root) cd /var/log/nginx/
+(app-root) ls -al
+total 20
+drwxrwx--x. 1 default root 4096 Sep 17 17:18 .
+drwxr-xr-x. 1 root    root 4096 Sep 17 15:13 ..
+lrwxrwxrwx. 1 default root   11 Sep 17 17:18 access.log -> /dev/stdout
+-rw-r--r--. 1 default root  132 Sep 17 17:18 codeserver.access.log
+lrwxrwxrwx. 1 default root   11 Sep 17 17:18 error.log -> /dev/stderr
+```
+
+```
+lrwxrwxrwx. 1 root root 15 Sep 12 11:24 /dev/stderr -> /proc/self/fd/2
+l-wx------. 1 default root 64 Sep 17 17:38 /proc/self/fd/2 -> pipe:[9523152]
+ls: cannot access '/dev/pts/0': No such file or directory
+```
+
+On a regular Linux machine, we have `/dev/pts/0`
+
+```
+lrwx------. 1 default root 64 Sep 17 17:34 /proc/self/fd/2 -> /dev/pts/0
+```
+
+To solve this, we need to either stop doing the symlinks for nginx, or replace /proc/self/fd/2 with something that nginx can reopen.

codeserver/ubi9-python-3.12/test/probe_check.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+"""
+Reproduce kubelet httpGet probe logic for the supplied path.
+Exits 0 if the probe will eventually pass, 1 otherwise.
+"""
+import sys
+import time
+from http.client import HTTPConnection, HTTPSConnection
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Type
+
+# probe constants from the odh-dashboard Deployment
+# https://github.com/opendatahub-io/odh-dashboard/blob/2.4.0-release/backend/src/utils/notebookUtils.ts#L310-L332
+INITIAL_DELAY = 10
+PERIOD = 5
+TIMEOUT = 1
+FAIL_THRESH = 3
+SUCCESS_THRESH = 1
+
+
+def probe_once(connection_factory: Type[HTTPConnection], host: str, port: int, path: str) -> tuple[bool, str]:
+    conn = None
+    try:
+        conn = connection_factory(host, port, timeout=TIMEOUT)
+        conn.request("GET", path)
+        resp = conn.getresponse()
+        return 200 <= resp.status < 400, str(resp.status)
+    except Exception as e:
+        return False, str(e)
+    finally:
+        try:
+            if conn:
+                conn.close()
+        except Exception:
+            pass
+
+
+def main(namespace, name, port) -> int:
+    path = f"/notebook/{namespace}/{name}/api"
+
+    # kubelet waits initialDelaySeconds before the first probe
+    time.sleep(INITIAL_DELAY)
+
+    consecutive_failures = 0
+    consecutive_successes = 0
+
+    while True:
+        ok, err = probe_once(HTTPConnection, "127.0.0.1", int(port), path)
+        print("probe_once:", ok, err)
+
+        if ok:
+            consecutive_successes += 1
+            consecutive_failures = 0
+            if consecutive_successes >= SUCCESS_THRESH:
+                return 0
+        else:
+            consecutive_failures += 1
+            consecutive_successes = 0
+            if consecutive_failures >= FAIL_THRESH:
+                return 1
+
+        time.sleep(PERIOD)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 4:
+        print("usage: probe_check.py <namespace> <name> <notebook-port>", file=sys.stderr)
+        sys.exit(2)
+
+    namespace, name, port = sys.argv[1:]
+    sys.exit(main(namespace, name, port))

codeserver/ubi9-python-3.12/test/test_startup.py

@@ -0,0 +1,96 @@
+import os
+import select
+import subprocess
+import sys
+import threading
+import unittest
+from pathlib import Path
+
+try:
+    import pytest
+except ImportError:
+
+    class pytest:
+        class _Mark:
+            @classmethod
+            def __getattr__(cls, item):
+                return lambda x: x
+
+        mark = _Mark()
+
+
+import probe_check
+
+
+@pytest.mark.buildonlytest
+class TestStartup(unittest.TestCase):
+    def setUp(self):
+        self.project_name = "projectName"
+        self.notebook_id = "notebookId"
+        self.translated_username = "sometranslatedUsername"
+        self.origin = "https://origin"
+        self.root_dir = Path(__file__).parent
+
+    def get_notebook_args_configuration(self) -> str:
+        expected_args = " ".join(
+            [
+                "--ServerApp.port=8888",
+                "--ServerApp.token=''",
+                "--ServerApp.password=''",
+                f"--ServerApp.base_url=/notebook/{self.project_name}/{self.notebook_id}",
+                "--ServerApp.quit_button=False",
+                f'--ServerApp.tornado_settings={{"user":"{self.translated_username}","hub_host":"{self.origin}","hub_prefix":"/projects/{self.project_name}"}}',
+            ]
+        )
+        return expected_args
+
+    def get_environment_variables(self) -> dict[str, str]:
+        notebook_args = self.get_notebook_args_configuration()
+        nb_prefix = f"/notebook/{self.project_name}/{self.notebook_id}"
+        return {
+            # (outdated?) https://github.com/opendatahub-io/odh-dashboard/blob/2.4.0-release/backend/src/utils/notebookUtils.ts#L284-L293
+            # https://github.com/opendatahub-io/odh-dashboard/blob/1d5a9065c10acc4706b84b06c67f27f16cf6dee7/frontend/src/api/k8s/notebooks.ts#L157-L170
+            "NOTEBOOK_ARGS": notebook_args,
+            # NB_PREFIX is set by notebook-controller and codeserver scripting depends on it
+            # https://github.com/opendatahub-io/kubeflow/blob/f924a96375988fe3801db883e99ce9ed1ab5939c/components/notebook-controller/controllers/notebook_controller.go#L417
+            "NB_PREFIX": nb_prefix,
+        }
+
+    def test_codeserver_startup(self):
+        env = self.get_environment_variables()
+        command = ["/opt/app-root/bin/run-code-server.sh"]
+        p = subprocess.Popen(
+            command,
+            # KFLUXSPRT-5139: https://redhat-internal.slack.com/archives/C04PZ7H0VA8/p1758128206734419
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env={**os.environ, **env},
+        )
+        thr = threading.Thread(target=pump_stream, args=(p,), daemon=True)
+        thr.start()
+
+        try:
+            ret = probe_check.main(self.project_name, self.notebook_id, "8888")
+            self.assertEqual(ret, 0, "Probe check should return 0")
+            self.assertEqual(p.poll(), None, "Code server process should still be running")
+        finally:
+            p.terminate()
+            p.wait()
+
+        thr.join()
+
+
+def pump_stream(process: subprocess.Popen) -> None:
+    """Copy everything from stream -> stdout."""
+    # this may stop pumping even if there's something still buffered, so be it
+    while process.returncode is None:
+        rl, _, _ = select.select([process.stdout], [], [], 1)
+        if rl:
+            chunk = process.stdout.read1()
+            sys.stdout.buffer.write(chunk)
+            sys.stdout.buffer.flush()
+    process.stdout.close()
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

pytest.ini

@@ -21,3 +21,4 @@ markers =
     openshift: requires openshift to run,
     cuda: requires cuda to run,
     rocm: requires rocm to run,
+    buildonlytest: runs inside docker build,