Removed create-cert init containers & side car container

Author: Bobbins228

pyproject.toml

@@ -48,6 +48,7 @@ pytest-timeout = "2.2.0"
 [tool.pytest.ini_options]
 filterwarnings = [
     "ignore::DeprecationWarning:pkg_resources",
+    "ignore::DeprecationWarning",
     "ignore:pkg_resources is deprecated as an API:DeprecationWarning",
 ]
 markers = [

src/codeflare_sdk.egg-info/PKG-INFO

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
-Name: codeflare-sdk
+Name: codeflare_sdk
 Version: 0.0.0
 License-File: LICENSE

src/codeflare_sdk.egg-info/SOURCES.txt

@@ -13,11 +13,9 @@ src/codeflare_sdk/cluster/cluster.py
 src/codeflare_sdk/cluster/config.py
 src/codeflare_sdk/cluster/model.py
 src/codeflare_sdk/job/__init__.py
-src/codeflare_sdk/job/jobs.py
 src/codeflare_sdk/job/ray_jobs.py
 src/codeflare_sdk/utils/__init__.py
 src/codeflare_sdk/utils/generate_cert.py
 src/codeflare_sdk/utils/generate_yaml.py
 src/codeflare_sdk/utils/kube_api_helpers.py
-src/codeflare_sdk/utils/openshift_oauth.py
 src/codeflare_sdk/utils/pretty_print.py

src/codeflare_sdk/templates/base-template.yaml

@@ -117,20 +117,7 @@ spec:
                           - "aw-kuberay"
                 containers:
                 # The Ray head pod
-                - env:
-                  - name: MY_POD_IP
-                    valueFrom:
-                      fieldRef:
-                        fieldPath: status.podIP
-                  - name: RAY_USE_TLS
-                    value: "0"
-                  - name: RAY_TLS_SERVER_CERT
-                    value: /home/ray/workspace/tls/server.crt
-                  - name: RAY_TLS_SERVER_KEY
-                    value: /home/ray/workspace/tls/server.key
-                  - name: RAY_TLS_CA_CERT
-                    value: /home/ray/workspace/tls/ca.crt
-                  name: ray-head
+                - name: ray-head
                   image: quay.io/project-codeflare/ray:latest-py39-cu118
                   imagePullPolicy: Always
                   ports:
@@ -172,30 +159,7 @@ spec:
                   - mountPath: /etc/ssl/certs/odh-ca-bundle.crt
                     name: odh-ca-cert
                     subPath: odh-ca-bundle.crt
-                initContainers:
-                - command:
-                  - sh
-                  - -c
-                  - cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-deployment-name-$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).server-name">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext
-                  image: quay.io/project-codeflare/ray:latest-py39-cu118
-                  name: create-cert
-                  # securityContext:
-                  #   runAsUser: 1000
-                  #   runAsGroup: 1000
-                  volumeMounts:
-                  - name: ca-vol
-                    mountPath: "/home/ray/workspace/ca"
-                    readOnly: true
-                  - name: server-cert
-                    mountPath: "/home/ray/workspace/tls"
-                    readOnly: false
                 volumes:
-                - name: ca-vol
-                  secret:
-                    secretName: ca-secret-deployment-name
-                  optional: false
-                - name: server-cert
-                  emptyDir: {}
                 - name: odh-trusted-ca-cert
                   configMap:
                     name: odh-trusted-ca-bundle
@@ -250,40 +214,9 @@ spec:
                           operator: In
                           values:
                           - "aw-kuberay"
-                initContainers:
-                # the env var $RAY_IP is set by the operator if missing, with the value of the head service name
-                - name: create-cert
-                  image: quay.io/project-codeflare/ray:latest-py39-cu118
-                  command:
-                  - sh
-                  - -c
-                  - cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext
-                  # securityContext:
-                  #   runAsUser: 1000
-                  #   runAsGroup: 1000
-                  volumeMounts:
-                  - name: ca-vol
-                    mountPath: "/home/ray/workspace/ca"
-                    readOnly: true
-                  - name: server-cert
-                    mountPath: "/home/ray/workspace/tls"
-                    readOnly: false
                 containers:
                 - name: machine-learning # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name',  or '123-abc'
                   image: quay.io/project-codeflare/ray:latest-py39-cu118
-                  env:
-                  - name: MY_POD_IP
-                    valueFrom:
-                      fieldRef:
-                        fieldPath: status.podIP
-                  - name: RAY_USE_TLS
-                    value: "0"
-                  - name: RAY_TLS_SERVER_CERT
-                    value: /home/ray/workspace/tls/server.crt
-                  - name: RAY_TLS_SERVER_KEY
-                    value: /home/ray/workspace/tls/server.key
-                  - name: RAY_TLS_CA_CERT
-                    value: /home/ray/workspace/tls/ca.crt
                   # environment variables to set in the container.Optional.
                   # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/
                   lifecycle:
@@ -319,12 +252,6 @@ spec:
                     name: odh-ca-cert
                     subPath: odh-ca-bundle.crt
                 volumes:
-                - name: ca-vol
-                  secret:
-                    secretName: ca-secret-deployment-name
-                  optional: false
-                - name: server-cert
-                  emptyDir: {}
                 - name: odh-trusted-ca-cert
                   configMap:
                     name: odh-trusted-ca-bundle

src/codeflare_sdk/utils/generate_yaml.py

@@ -85,20 +85,6 @@ def update_names(yaml, item, appwrapper_name, cluster_name, namespace):
     lower_meta["labels"]["workload.codeflare.dev/appwrapper"] = appwrapper_name
     lower_meta["name"] = cluster_name
     lower_meta["namespace"] = namespace
-    lower_spec = item.get("generictemplate", {}).get("spec")
-    if is_openshift_cluster():
-        cookie_secret_env_var = {
-            "name": "COOKIE_SECRET",
-            "valueFrom": {
-                "secretKeyRef": {
-                    "key": "cookie_secret",
-                    "name": f"{cluster_name}-oauth-config",
-                }
-            },
-        }
-        lower_spec["headGroupSpec"]["template"]["spec"]["containers"][0]["env"].append(
-            cookie_secret_env_var
-        )
 
 
 def update_labels(yaml, instascale, instance_types):
@@ -291,44 +277,13 @@ def update_ca_secret(ca_secret_item, cluster_name, namespace):
     data["ca.key"], data["ca.crt"] = generate_cert.generate_ca_cert(365)
 
 
-def enable_local_interactive(resources, cluster_name, namespace):  # pragma: no cover
-    from ..cluster.cluster import _get_ingress_domain
-
-    ca_secret_item = resources["resources"].get("GenericItems")[1]
+def enable_local_interactive(resources):  # pragma: no cover
     item = resources["resources"].get("GenericItems")[0]
-    update_ca_secret(ca_secret_item, cluster_name, namespace)
-    # update_ca_secret_volumes
-    item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"]["volumes"][0][
-        "secret"
-    ]["secretName"] = f"ca-secret-{cluster_name}"
-    item["generictemplate"]["spec"]["workerGroupSpecs"][0]["template"]["spec"][
-        "volumes"
-    ][0]["secret"]["secretName"] = f"ca-secret-{cluster_name}"
-    # update_tls_env
-    item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"]["containers"][
-        0
-    ]["env"][1]["value"] = "1"
-    item["generictemplate"]["spec"]["workerGroupSpecs"][0]["template"]["spec"][
-        "containers"
-    ][0]["env"][1]["value"] = "1"
-    # update_init_container
-    command = item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"][
-        "initContainers"
-    ][0].get("command")[2]
-
-    command = command.replace("deployment-name", cluster_name)
-
-    domain = ""  ## FIX - We can't retrieve ingress domain - move init container to CFO
 
-    command = command.replace("server-name", domain)
     item["generictemplate"]["metadata"]["annotations"][
         "sdk.codeflare.dev/local_interactive"
     ] = "True"
 
-    item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"][
-        "initContainers"
-    ][0].get("command")[2] = command
-
 
 def del_from_list_by_name(l: list, target: typing.List[str]) -> list:
     return [x for x in l if x["name"] not in target]
@@ -392,75 +347,6 @@ def write_user_appwrapper(user_yaml, output_file_name):
     print(f"Written to: {output_file_name}")
 
 
-def enable_openshift_oauth(user_yaml, cluster_name, namespace):
-    config_check()
-    k8_client = api_config_handler() or client.ApiClient()
-    tls_mount_location = "/etc/tls/private"
-    oauth_port = 8443
-    oauth_sa_name = f"{cluster_name}-oauth-proxy"
-    tls_secret_name = f"{cluster_name}-proxy-tls-secret"
-    tls_volume_name = "proxy-tls-secret"
-    port_name = "oauth-proxy"
-    oauth_sidecar = _create_oauth_sidecar_object(
-        namespace,
-        tls_mount_location,
-        oauth_port,
-        oauth_sa_name,
-        tls_volume_name,
-        port_name,
-    )
-    tls_secret_volume = client.V1Volume(
-        name=tls_volume_name,
-        secret=client.V1SecretVolumeSource(secret_name=tls_secret_name),
-    )
-    # allows for setting value of Cluster object when initializing object from an existing AppWrapper on cluster
-    user_yaml["metadata"]["annotations"] = user_yaml["metadata"].get("annotations", {})
-    ray_headgroup_pod = user_yaml["spec"]["resources"]["GenericItems"][0][
-        "generictemplate"
-    ]["spec"]["headGroupSpec"]["template"]["spec"]
-    ray_headgroup_pod["serviceAccount"] = oauth_sa_name
-    ray_headgroup_pod["volumes"] = ray_headgroup_pod.get("volumes", [])
-
-    # we use a generic api client here so that the serialization function doesn't need to be mocked for unit tests
-    ray_headgroup_pod["volumes"].append(
-        client.ApiClient().sanitize_for_serialization(tls_secret_volume)
-    )
-    ray_headgroup_pod["containers"].append(
-        client.ApiClient().sanitize_for_serialization(oauth_sidecar)
-    )
-
-
-def _create_oauth_sidecar_object(
-    namespace: str,
-    tls_mount_location: str,
-    oauth_port: int,
-    oauth_sa_name: str,
-    tls_volume_name: str,
-    port_name: str,
-) -> client.V1Container:
-    return client.V1Container(
-        args=[
-            f"--https-address=:{oauth_port}",
-            "--provider=openshift",
-            f"--openshift-service-account={oauth_sa_name}",
-            "--upstream=http://localhost:8265",
-            f"--tls-cert={tls_mount_location}/tls.crt",
-            f"--tls-key={tls_mount_location}/tls.key",
-            "--cookie-secret=$(COOKIE_SECRET)",
-            f'--openshift-delegate-urls={{"/":{{"resource":"pods","namespace":"{namespace}","verb":"get"}}}}',
-        ],
-        image="registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366",
-        name="oauth-proxy",
-        ports=[client.V1ContainerPort(container_port=oauth_port, name=port_name)],
-        resources=client.V1ResourceRequirements(limits=None, requests=None),
-        volume_mounts=[
-            client.V1VolumeMount(
-                mount_path=tls_mount_location, name=tls_volume_name, read_only=True
-            )
-        ],
-    )
-
-
 def get_default_kueue_name(namespace: str):
     # If the local queue is set, use it. Otherwise, try to use the default queue.
     try:
@@ -620,12 +506,13 @@ def generate_appwrapper(
     )
 
     if local_interactive:
-        enable_local_interactive(resources, cluster_name, namespace)
-    else:
-        disable_raycluster_tls(resources["resources"])
+        enable_local_interactive(resources)
 
-    if is_openshift_cluster():
-        enable_openshift_oauth(user_yaml, cluster_name, namespace)
+    # else:
+    #     disable_raycluster_tls(resources["resources"])
+
+    ca_secret_item = resources["resources"].get("GenericItems")[1]
+    update_ca_secret(ca_secret_item, cluster_name, namespace)
 
     directory_path = os.path.expanduser("~/.codeflare/resources/")
     outfile = os.path.join(directory_path, appwrapper_name + ".yaml")

tests/test-case-no-mcad.yamls

@@ -41,20 +41,7 @@ spec:
                   values:
                   - unit-test-cluster-ray
         containers:
-        - env:
-          - name: MY_POD_IP
-            valueFrom:
-              fieldRef:
-                fieldPath: status.podIP
-          - name: RAY_USE_TLS
-            value: '0'
-          - name: RAY_TLS_SERVER_CERT
-            value: /home/ray/workspace/tls/server.crt
-          - name: RAY_TLS_SERVER_KEY
-            value: /home/ray/workspace/tls/server.key
-          - name: RAY_TLS_CA_CERT
-            value: /home/ray/workspace/tls/ca.crt
-          image: quay.io/project-codeflare/ray:latest-py39-cu118
+        - image: quay.io/project-codeflare/ray:latest-py39-cu118
           imagePullPolicy: Always
           lifecycle:
             preStop:
@@ -81,6 +68,12 @@ spec:
               memory: 8G
               nvidia.com/gpu: 0
           volumeMounts:
+          - mountPath: /home/ray/workspace/ca
+            name: ca-vol
+            readOnly: true
+          - mountPath: /home/ray/workspace/tls
+            name: server-cert
+            readOnly: true
           - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt
             name: odh-trusted-ca-cert
             subPath: odh-trusted-ca-bundle.crt
@@ -136,20 +129,7 @@ spec:
                   values:
                   - unit-test-cluster-ray
         containers:
-        - env:
-          - name: MY_POD_IP
-            valueFrom:
-              fieldRef:
-                fieldPath: status.podIP
-          - name: RAY_USE_TLS
-            value: '0'
-          - name: RAY_TLS_SERVER_CERT
-            value: /home/ray/workspace/tls/server.crt
-          - name: RAY_TLS_SERVER_KEY
-            value: /home/ray/workspace/tls/server.key
-          - name: RAY_TLS_CA_CERT
-            value: /home/ray/workspace/tls/ca.crt
-          image: quay.io/project-codeflare/ray:latest-py39-cu118
+        - image: quay.io/project-codeflare/ray:latest-py39-cu118
           lifecycle:
             preStop:
               exec:
@@ -168,6 +148,12 @@ spec:
               memory: 5G
               nvidia.com/gpu: 7
           volumeMounts:
+          - mountPath: /home/ray/workspace/ca
+            name: ca-vol
+            readOnly: true
+          - mountPath: /home/ray/workspace/tls
+            name: server-cert
+            readOnly: true
           - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt
             name: odh-trusted-ca-cert
             subPath: odh-trusted-ca-bundle.crt
@@ -197,3 +183,14 @@ spec:
             name: odh-trusted-ca-bundle
             optional: true
           name: odh-ca-cert
+---
+apiVersion: v1
+data:
+  ca.crt: ca-field
+  ca.key: ca-field
+kind: Secret
+metadata:
+  labels:
+    odh-ray-cluster-service: unit-test-cluster-ray-head-svc
+  name: ca-secret-unit-test-cluster-ray
+  namespace: ns