CLOUDP-342878 - Add --remove-destination to cp (#407)

# Summary

Adding the `--remove-destination` flag to `cp` in `setup-agent-files.sh`
will prevent the agent container from getting stuck whenever the agent
or the utilities container is restarted.
If the utilities container is restarted, then the pid of the utilities
marker will change. This means that the symlink for the probes are
pointing to the filesystem of the old pid, therefore a dangling symlink.
When the agent container is restarted, `cp` will not be able to
overwrite the symlink without this flag.

This change will require re-releasing all agent images.

## Proof of Work
Before the change
<img width="1488" height="64" alt="image"
src="https://github.com/user-attachments/assets/8cc664b9-491a-445b-b71c-2f9b89fa844a"
/>

After the change. The containers are ready even if they were restarted
<img width="1444" height="83" alt="image"
src="https://github.com/user-attachments/assets/26f4857a-9f6e-4eca-9e74-82acd5d9f145"
/>


## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: lucian-tosa

docker/mongodb-agent/setup-agent-files.sh

@@ -8,8 +8,8 @@ SCRIPTS_DIR="/opt/scripts"
 # readiness always returns failure
 setup_dummy_probes() {
   echo "Setting up dummy probe scripts..."
-  cp /usr/local/bin/dummy-probe.sh "$SCRIPTS_DIR/probe.sh"
-  cp /usr/local/bin/dummy-readinessprobe "$SCRIPTS_DIR/readinessprobe"
+  cp --remove-destination /usr/local/bin/dummy-probe.sh "$SCRIPTS_DIR/probe.sh"
+  cp --remove-destination /usr/local/bin/dummy-readinessprobe "$SCRIPTS_DIR/readinessprobe"
   echo "Dummy probe scripts ready"
 }
 

scripts/evergreen/e2e/dump_diagnostic_information.sh

@@ -126,29 +126,26 @@ dump_pod_logs() {
         kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent.log" "logs/${prefix}${pod}-agent.log" &> /dev/null
         kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent-verbose.log" "logs/${prefix}${pod}-monitoring-agent-verbose.log" &> /dev/null
         kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/monitoring-agent.log" "logs/${prefix}${pod}-monitoring-agent.log" &> /dev/null
-        kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-monitoring" > "logs/${prefix}${pod}-monitoring-agent-stdout.log" || true
-        kubectl logs -n "${namespace}" "${pod}" -c "mongod" > "logs/${prefix}${pod}-mongod-container.log" || true
-        kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent" > "logs/${prefix}${pod}-mongodb-agent-container.log" || true
         kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true
 
         # note that this file may get empty if the logs have already grew too much - seems it's better to have it explicitly empty then just omit
         kubectl logs -n "${namespace}" "${pod}" | jq -c -r 'select( .logType == "agent-launcher-script") | .contents' 2> /dev/null > "logs/${prefix}${pod}-launcher.log"
-    else
-        # for all other pods we want each log per container from kubectl
-        for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do
-            echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log"
-            kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log"
+    fi
 
-            # Check if the container has restarted by examining its restart count
-            restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}")
+    for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do
+        echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log"
+        kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log"
 
-            if [ "${restartCount}" -gt 0 ]; then
-                echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log"
-                kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true
-            fi
+        # Check if the container has restarted by examining its restart count
+        restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}")
+
+        if [ "${restartCount}" -gt 0 ]; then
+            echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log"
+            kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true
+        fi
+
+    done
 
-        done
-    fi
 
     if kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-stderr.log &>/dev/null; then
         kubectl cp "${namespace}/${pod}:/var/log/mongodb-mms-automation/automation-agent-stderr.log" "logs/${prefix}${pod}-agent-stderr.log" &> /dev/null