Merge pull request #41 from osusec/dr/deploy-kubernetes

Implement challenge Kubernetes resource deployment

Author: detjensrobert

Cargo.lock

@@ -20,8 +20,8 @@ figment = { version = "0.10.19", features = ["env", "yaml", "test"] }
 zip = { version = "2.2.2", default-features = false, features = ["deflate"] }
 
 # kubernetes:
-kube = { version = "0.91.0", features = ["runtime", "derive"] }
-k8s-openapi = { version = "0.22.0", features = ["latest"] }
+kube = { version = "0.99.0", features = ["runtime", "derive"] }
+k8s-openapi = { version = "0.24.0", features = ["latest"] }
 tokio = { version = "1.38.0", features = ["rt", "macros"] }
 http = { version = "1.2", default-features = false }
 
@@ -33,7 +33,7 @@ rust-s3 = { version = "0.35.1", default-features = false, features = [
   "fail-on-err",
   "tokio-rustls-tls",
 ] }
-minijinja = "2.6.0"
+minijinja = { version = "2.6.0", features = ["json"] }
 duct = "0.13.7"
 fastrand = "2.3.0"
 

Cargo.toml

@@ -1,21 +1,14 @@
-{% set chal = whatever -%}
-{% set pod = whatever -%}
-{% set slug = chal.name | slugify -%}
-
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: "rcds-{{ slug }}-{{ pod.name }}"
   namespace: "rcds-{{ slug }}"
   annotations:
+    app.kubernetes.io/managed-by: rcds
     rctf/challenge: "{{ chal.name }}"
     rctf/category: "{{ chal.category }}"
-    rctf/description: "{{ chal.description }}"
-    rctf/flag: "{{ chal.flag }}"
-    rctf/points: "{{ chal.points }}"
-    rctf/files: "{{ chal.points }}"
-    app.kubernetes.io/managed-by: rcds
+    rctf/challenge-pod: "{{ pod.name }}"
 spec:
   selector:
     matchLabels:
@@ -28,20 +21,29 @@ spec:
     spec:
       containers:
         - name: "{{ pod.name }}"
-          image: "{{ pod.image }}"
+          image: "{{ pod_image }}"
           ports:
             {% for p in pod.ports -%}
             - containerPort: {{ p.internal }}
+            {%- else %}
+            []
             {%- endfor %}
+
           {% if pod.env -%}
           env:
             {% for k, v in pod.env -%}
             - { name: "{{ k }}", value: "{{ v }}" }
+            {%- else %}
+            []
             {%- endfor %}
           {%- endif %}
+
+          {% if pod.resources -%}
           resources:
-            requests: {{ pod.resources | json_encode() | safe }}
-            limits: {{ pod.resources | json_encode() | safe }}
+            {# TODO: use defaults from rcds config -#}
+            requests: {{ pod.resources | tojson }}
+            limits: {{ pod.resources | tojson }}
+          {%- endif %}
 
       # don't give chal pods k8s api tokens
       automountServiceAccountToken: false

src/asset_files/challenge_templates/deployment.yaml.j2

@@ -1,13 +1,8 @@
-{% set chal = whatever -%}
-{% set pod = current from chal -%}
-{% set rcds = global rcds yaml -%}
-{% set slug = chal.name | slugify -%}
-
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: "rcds-{{ slug }}-{{ pod.name }}"
+  name: "rcds-{{ slug }}-{{ pod.name }}-http"
   namespace: "rcds-{{ slug }}"
   annotations:
     app.kubernetes.io/managed-by: rcds
@@ -16,7 +11,7 @@ spec:
     rctf/part-of: "{{ slug }}-{{ pod.name }}"
   ports:
     # host service at same port as container
-    {% for p in pod.ports -%}
+    {% for p in http_ports -%}
     - port: {{ p.internal }}
       targetPort: {{ p.internal }}
     {%- endfor %}
@@ -30,18 +25,17 @@ metadata:
   annotations:
     app.kubernetes.io/managed-by: rcds
 spec:
-  ingressClassName: beaverctf-nginx
+  ingressClassName: beavercds
   rules:
-  {% for p in pod.ports | filter(attribute="expose.http") -%}
-    - host: "{{ p.expose.http }}.{{ rcds.domain }}"
+  {%- for p in http_ports %}
+    - host: "{{ p.expose.http }}.{{ domain }}"
       http:
         paths:
         - pathType: Prefix
           path: "/"
           backend:
             service:
-              name: "rcds-{{ slug }}-{{ pod.name }}"
+              name: "rcds-{{ slug }}-{{ pod.name }}-http"
               port:
-                # find first pod http port
                 number: {{ p.internal }}
-  {%- endfor %}
+  {% endfor -%}

src/asset_files/challenge_templates/http.yaml.j2

@@ -1,10 +1,11 @@
-{% set chal = whatever -%}
-{% set slug = chal.name | slugify -%}
-
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
   name: rcds-{{ slug }}
   annotations:
     app.kubernetes.io/managed-by: rcds
+    rctf/challenge: "{{ chal.name }}"
+    rctf/category: "{{ chal.category }}"
+    rctf/description: |
+      {{ chal.description | indent(6) }}

src/asset_files/challenge_templates/namespace.yaml.j2

@@ -1,26 +1,21 @@
-{% set chal = whatever -%}
-{% set pod = current from chal -%}
-{% set rcds = global rcds yaml -%}
-{% set slug = chal.name | slugify -%}
-
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: "rcds-{{ slug }}-{{ pod.name }}"
+  name: "rcds-{{ slug }}-{{ pod.name }}-tcp"
   namespace: "rcds-{{ slug }}"
   annotations:
     app.kubernetes.io/managed-by: rcds
     # still use separate domain for these, since exposed LoadBalancer services
     # will all have different ips from each other
-    external-dns.alpha.kubernetes.io/hostname: "{{ slug }}.{{ rcds.domain }}"
+    external-dns.alpha.kubernetes.io/hostname: "{{ slug }}.{{ domain }}"
 spec:
   type: LoadBalancer
   selector:
     rctf/part-of: "{{ slug }}-{{ pod.name }}"
   ports:
-  {% for p in pod.ports | filter(attribute="expose.tcp") -%}
+  {%- for p in tcp_ports %}
     - port: {{ p.expose.tcp }}
       targetPort: {{ p.internal }}
       protocol: TCP
-  {%- endfor %}
+  {% endfor -%}

src/asset_files/challenge_templates/tcp.yaml.j2

@@ -21,7 +21,7 @@ txtPrefix: "k8s-owner."
 
 extraArgs:
   # ignore any services with internal ips
-  exclude-target-net: "10.0.0.0/8"
+  #exclude-target-net: "10.0.0.0/8"
   # special character replacement
   txt-wildcard-replacement: star
 

src/asset_files/setup_manifests/external-dns.helm.yaml.j2

@@ -79,7 +79,7 @@ pub async fn extract_asset(
 
             let name = format!(
                 "asset-container-{}-{}-{}",
-                chal.directory.to_string_lossy().replace("/", "-"),
+                chal.slugify(),
                 container_name,
                 // include random discriminator to avoid name collisions
                 repeat_with(fastrand::alphanumeric)

src/builder/artifacts.rs

@@ -5,11 +5,17 @@ use std::sync::OnceLock;
 use anyhow::{anyhow, bail, Context, Error, Result};
 use bollard;
 use futures::TryFutureExt;
+use k8s_openapi::api::{
+    apps::v1::Deployment,
+    core::v1::{Pod, Service},
+    networking::v1::Ingress,
+};
 use kube::{
     self,
-    api::{DynamicObject, GroupVersionKind, TypeMeta},
+    api::{DynamicObject, GroupVersionKind, Patch, PatchParams},
     core::ResourceExt,
-    discovery::{ApiCapabilities, ApiResource, Discovery, Scope},
+    discovery::{ApiCapabilities, ApiResource},
+    runtime::{conditions, wait::await_condition},
 };
 use s3;
 use simplelog::*;
@@ -201,3 +207,141 @@ pub async fn kube_api_for(
         Ok(kube::Api::default_namespaced_with(client, &resource))
     }
 }
+
+/// Apply multi-document manifest file, return created resources
+pub async fn apply_manifest_yaml(
+    client: &kube::Client,
+    manifest: &str,
+) -> Result<Vec<DynamicObject>> {
+    // set ourself as the owner for managed fields
+    // https://kubernetes.io/docs/reference/using-api/server-side-apply/#managers
+    let pp = PatchParams::apply("beavercds").force();
+
+    let mut results = vec![];
+
+    // this manifest has multiple documents (crds, deployment)
+    for yaml in multidoc_deserialize(manifest)? {
+        let obj: DynamicObject = serde_yml::from_value(yaml)?;
+        debug!(
+            "applying resource {} {}",
+            obj.types.clone().unwrap_or_default().kind,
+            obj.name_any()
+        );
+
+        let obj_api = kube_api_for(&obj, client.clone()).await?;
+        match obj_api
+            // patch is idempotent and will create if not present
+            .patch(&obj.name_any(), &pp, &Patch::Apply(&obj))
+            .await
+        {
+            Ok(d) => {
+                results.push(d);
+                Ok(())
+            }
+            // if error is from cluster api, mark it as such
+            Err(kube::Error::Api(ae)) => {
+                // Err(kube::Error::Api(ae).into())
+                Err(anyhow!(ae).context("error from cluster when deploying"))
+            }
+            // other errors could be anything
+            Err(e) => Err(anyhow!(e)).context("unknown error when deploying"),
+        }?;
+    }
+
+    Ok(results)
+}
+
+/// Deserialize multi-document yaml string into a Vec of the documents
+fn multidoc_deserialize(data: &str) -> Result<Vec<serde_yml::Value>> {
+    use serde::Deserialize;
+
+    let mut docs = vec![];
+    for de in serde_yml::Deserializer::from_str(data) {
+        match serde_yml::Value::deserialize(de)? {
+            // discard any empty documents (e.g. from trailing ---)
+            serde_yml::Value::Null => (),
+            not_null => docs.push(not_null),
+        };
+    }
+    Ok(docs)
+
+    // // deserialize all chunks
+    // serde_yml::Deserializer::from_str(data)
+    //     .map(serde_yml::Value::deserialize)
+    //     // discard any empty documents (e.g. from trailing ---)
+    //     .filter_ok(|val| val != &serde_yml::Value::Null)
+    //     // coerce errors to Anyhow
+    //     .map(|r| r.map_err(|e| e.into()))
+    //     .collect()
+}
+
+/// Check the status of the passed object and wait for it to become ready.
+///
+/// This function does not provide a timeout. Callers will need to wrap this with a timeout instead.
+pub async fn wait_for_status(client: &kube::Client, object: &DynamicObject) -> Result<()> {
+    debug!(
+        "waiting for ok status for {} {}",
+        object.types.clone().unwrap_or_default().kind,
+        object.name_any()
+    );
+
+    // handle each separate object type differently
+    match object.types.clone().unwrap_or_default().kind.as_str() {
+        // wait for Pod to become running
+        "Pod" => {
+            let api = kube::Api::namespaced(client.clone(), &object.namespace().unwrap());
+            let x = await_condition(api, &object.name_any(), conditions::is_pod_running()).await?;
+        }
+
+        // wait for Deployment to complete rollout
+        "Deployment" => {
+            let api = kube::Api::namespaced(client.clone(), &object.namespace().unwrap());
+            await_condition(
+                api,
+                &object.name_any(),
+                conditions::is_deployment_completed(),
+            )
+            .await?;
+        }
+
+        // wait for Ingress to get IP from ingress controller
+        "Ingress" => {
+            let api = kube::Api::namespaced(client.clone(), &object.namespace().unwrap());
+            await_condition(
+                api,
+                &object.name_any(),
+                conditions::is_ingress_provisioned(),
+            )
+            .await?;
+        }
+
+        // wait for LoadBalancer service to get IP
+        "Service" => {
+            let api = kube::Api::namespaced(client.clone(), &object.namespace().unwrap());
+            let svc: Service = api.get(&object.name_any()).await?;
+
+            // we only care about checking LoadBalancer-type services, return Ok
+            // for any non-LB services
+            //
+            // TODO: do we care about NodePorts? don't need to check any atm
+            if svc.spec.unwrap_or_default().type_ != Some("LoadBalancer".to_string()) {
+                trace!(
+                    "not checking status for internal service {}",
+                    object.name_any()
+                );
+                return Ok(());
+            }
+
+            await_condition(
+                api,
+                &object.name_any(),
+                conditions::is_service_loadbalancer_provisioned(),
+            )
+            .await?;
+        }
+
+        other => trace!("not checking status for resource type {other}"),
+    };
+
+    Ok(())
+}