Owls 103821 - Check for a marker file in the emptyDir to detect if aux image init has already completed (#3595)

* OWLS-103821 - Fix for MII AI fails to restart after k8s shutdown and restarted.

Author: ankedia

operator/src/main/resources/scripts/auxImage.sh

@@ -33,28 +33,36 @@ UNKNOWN_SHELL=true
 checkEnv AUXILIARY_IMAGE_TARGET_PATH AUXILIARY_IMAGE_CONTAINER_NAME || exit 1
 
 if [[ "$AUXILIARY_IMAGE_CONTAINER_NAME" == "operator-aux-container"* ]]; then
-  initAuxiliaryImage > /tmp/auxiliaryImage.out 2>&1
-  retval=$?
-  cat /tmp/auxiliaryImage.out
-
-  mkdir -p ${AUXILIARY_IMAGE_TARGET_PATH}/auxiliaryImageLogs
-  cp /tmp/auxiliaryImage.out ${AUXILIARY_IMAGE_TARGET_PATH}/auxiliaryImageLogs/${AUXILIARY_IMAGE_CONTAINER_NAME}.out
   sucFile="${AUXILIARY_IMAGE_TARGET_PATH}/auxiliaryImageLogs/${AUXILIARY_IMAGE_CONTAINER_NAME}.suc"
-  rm -f "$sucFile"
-  if [ $retval -eq 0 ]; then
-    echo $retval > "$sucFile"
+  if [ ! -f $sucFile ]; then
+    initAuxiliaryImage > /tmp/auxiliaryImage.out 2>&1
+    retval=$?
+    cat /tmp/auxiliaryImage.out
+
+    mkdir -p ${AUXILIARY_IMAGE_TARGET_PATH}/auxiliaryImageLogs
+    cp /tmp/auxiliaryImage.out ${AUXILIARY_IMAGE_TARGET_PATH}/auxiliaryImageLogs/${AUXILIARY_IMAGE_CONTAINER_NAME}.out
+    rm -f "$sucFile"
+    if [ $retval -eq 0 ]; then
+      echo $retval > "$sucFile"
+    fi
+  else
+    trace FINE "Auxiliary Image: Skipping initialization due to a previous successful initialization."
   fi
 elif [[ "$AUXILIARY_IMAGE_CONTAINER_NAME" == "compatibility-mode-operator-aux-container"* ]]; then
-  initCompatibilityModeInitContainersWithLegacyAuxImages > /tmp/compatibilityModeInitContainers.out 2>&1
-  retval=$?
-  cat /tmp/compatibilityModeInitContainers.out
-
-  mkdir -p "${AUXILIARY_IMAGE_TARGET_PATH}/${AUXILIARY_IMAGE_COMMAND_LOGS_DIR}"
-  cp /tmp/compatibilityModeInitContainers.out "${AUXILIARY_IMAGE_TARGET_PATH}/${AUXILIARY_IMAGE_COMMAND_LOGS_DIR}/${AUXILIARY_IMAGE_CONTAINER_NAME}.out"
   sucFile="${AUXILIARY_IMAGE_TARGET_PATH}/${AUXILIARY_IMAGE_COMMAND_LOGS_DIR}/${AUXILIARY_IMAGE_CONTAINER_NAME}.suc"
-  rm -f "$sucFile"
-  if [ $retval -eq 0 ]; then
-    echo $retval > "$sucFile"
+  if [ ! -f $sucFile ]; then
+    initCompatibilityModeInitContainersWithLegacyAuxImages > /tmp/compatibilityModeInitContainers.out 2>&1
+    retval=$?
+    cat /tmp/compatibilityModeInitContainers.out
+
+    mkdir -p "${AUXILIARY_IMAGE_TARGET_PATH}/${AUXILIARY_IMAGE_COMMAND_LOGS_DIR}"
+    cp /tmp/compatibilityModeInitContainers.out "${AUXILIARY_IMAGE_TARGET_PATH}/${AUXILIARY_IMAGE_COMMAND_LOGS_DIR}/${AUXILIARY_IMAGE_CONTAINER_NAME}.out"
+    rm -f "$sucFile"
+    if [ $retval -eq 0 ]; then
+      echo $retval > "$sucFile"
+    fi
+  else
+    trace FINE "Auxiliary Image: Skipping initialization due to a previous successful initialization."
   fi
 else
   trace SEVERE "Invalid auxiliary image container name '$AUXILIARY_IMAGE_CONTAINER_NAME'. " \