Backport owls-99679 to release/3.4 (#3168)

* backport owls-99679 to 3.4 branch

Co-authored-by: Ryan Eberhard <ryan.eberhard@oracle.com>

Author: alai8

documentation/domains/Domain.json

@@ -480,7 +480,7 @@
           "type": "string"
         },
         "introspectVersion": {
-          "description": "Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. See also `domains.spec.configuration.overrideDistributionStrategy`.",
+          "description": "Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. The introspectVersion value must be a valid label value in Kubernetes. See also `domains.spec.configuration.overrideDistributionStrategy`.",
           "type": "string"
         },
         "dataHome": {

documentation/domains/Domain.md

@@ -32,7 +32,7 @@ The specification of the operation of the WebLogic domain. Required.
 | `imagePullPolicy` | string | The image pull policy for the WebLogic Server image. Legal values are Always, Never, and IfNotPresent. Defaults to Always if image ends in :latest; IfNotPresent, otherwise. |
 | `imagePullSecrets` | Array of [Local Object Reference](k8s1.13.5.md#local-object-reference) | A list of image pull Secrets for the WebLogic Server image. |
 | `includeServerOutInPodLog` | Boolean | Specifies whether the server .out file will be included in the Pod's log. Defaults to true. |
-| `introspectVersion` | string | Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. See also `domains.spec.configuration.overrideDistributionStrategy`. |
+| `introspectVersion` | string | Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. The introspectVersion value must be a valid label value in Kubernetes. See also `domains.spec.configuration.overrideDistributionStrategy`. |
 | `livenessProbeCustomScript` | string | Full path of an optional liveness probe custom script for WebLogic Server instance pods. The existing liveness probe script `livenessProbe.sh` will invoke this custom script after the existing script performs its own checks. This element is optional and is for advanced usage only. Its value is not set by default. If the custom script fails with non-zero exit status, then pod will fail the liveness probe and Kubernetes will restart the container. If the script specified by this element value is not found, then it is ignored. |
 | `logHome` | string | The directory in a server's container in which to store the domain, Node Manager, server logs, server *.out, introspector .out, and optionally HTTP access log files if `httpAccessLogInLogHome` is true. Default is `/shared/logs/DOMAIN-UID`. Ignored if `logHomeEnabled` is false. |
 | `logHomeEnabled` | Boolean | Specifies whether the log home folder is enabled. Defaults to true if `domainHomeSourceType` is PersistentVolume; false, otherwise. |

documentation/domains/index.html

@@ -1401,7 +1401,7 @@
           "type": "string"
         },
         "introspectVersion": {
-          "description": "Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. See also `domains.spec.configuration.overrideDistributionStrategy`.",
+          "description": "Changes to this field cause the operator to repeat its introspection of the WebLogic domain configuration. Repeating introspection is required for the operator to recognize changes to the domain configuration, such as adding a new WebLogic cluster or Managed Server instance, to regenerate configuration overrides, or to regenerate the WebLogic domain home when the `domainHomeSourceType` is FromModel. Introspection occurs automatically, without requiring change to this field, when servers are first started or restarted after a full domain shut down. For the FromModel `domainHomeSourceType`, introspection also occurs when a running server must be restarted because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. The introspectVersion value must be a valid label value in Kubernetes. See also `domains.spec.configuration.overrideDistributionStrategy`.",
           "type": "string"
         },
         "dataHome": {

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java

@@ -830,7 +830,7 @@ void testDedicatedModeSameNamespaceScale() {
    * Update the introspectVersion of the domain resource using lifecycle script.
    * Refer to kubernetes/samples/scripts/domain-lifecycle/introspectDomain.sh
    * The usecase update the introspectVersion by passing different value to -i
-   * option (non-numeric, non-numeric with space, no value) and make sure that
+   * option (non-numeric, non-numeric with underscore and dash, no value) and make sure that
    * the introspectVersion is updated accordingly in both domain spec level
    * and server pod level.
    * It also verifies the introspector job is started/stopped and none of the
@@ -894,8 +894,8 @@ void testIntrospectDomainScript() {
     verifyIntrospectVersionLabelInPod();
 
     // use introspectDomain.sh to initiate introspection
-    logger.info("Initiate introspection with non numeric string with space");
-    introspectVersion = "My Version";
+    logger.info("Initiate introspection with non numeric string with underscore and dash");
+    introspectVersion = "My_Version-1";
     String extraParam2 = " -i " + "\"" + introspectVersion + "\"";
     assertDoesNotThrow(() -> executeLifecycleScript(INTROSPECT_DOMAIN_SCRIPT, extraParam2),
         String.format("Failed to run %s", INTROSPECT_DOMAIN_SCRIPT));

kubernetes/crd/domain-crd.yaml

@@ -5,7 +5,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    weblogic.sha256: bd0abf9c35b0f0fae26a2f0fa1a8e7e4ef15e54858054eece1411ebb8257622c
+    weblogic.sha256: daef2448066da223ba8cb86e31e2d711d9588eb736cf4558071113c396d95b69
   name: domains.weblogic.oracle
 spec:
   group: weblogic.oracle
@@ -6341,6 +6341,7 @@ spec:
                   full domain shut down. For the FromModel `domainHomeSourceType`,
                   introspection also occurs when a running server must be restarted
                   because of changes to any of the fields listed here: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted.
+                  The introspectVersion value must be a valid label value in Kubernetes.
                   See also `domains.spec.configuration.overrideDistributionStrategy`.'
                 type: string
               dataHome:

operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java

@@ -9,6 +9,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import javax.annotation.Nonnull;
 
 import io.kubernetes.client.openapi.models.V1ContainerState;
 import io.kubernetes.client.openapi.models.V1ContainerStateWaiting;
@@ -489,7 +490,8 @@ static OffsetDateTime createNextSteps(List<Step> nextSteps, Packet packet, V1Job
                 .map(V1ObjectMeta::getCreationTimestamp).orElse(OffsetDateTime.now());
         String lastIntrospectJobProcessedId = getLastIntrospectJobProcessedId(info);
 
-        if (isJobTimedout(info) || (isImagePullError(jobPodContainerWaitingReason)) || jobInitContainerImagePullError) {
+        if (isJobTimedout(info) || (isImagePullError(jobPodContainerWaitingReason))
+            || jobInitContainerImagePullError || isInProgressJobOutdated(job, info)) {
           jobStartTime = OffsetDateTime.now();
           packet.put(DOMAIN_INTROSPECT_REQUESTED, ReadDomainIntrospectorPodLogResponseStep.INTROSPECTION_FAILED);
           nextSteps.add(Step.chain(deleteDomainIntrospectorJobStep(null),
@@ -517,6 +519,26 @@ private static Boolean getjobInitContainerImagePullError(Packet packet) {
       return Optional.ofNullable(packet.<Boolean>getValue(ProcessingConstants.JOB_POD_INIT_CONTAINER_WAITING_REASON))
               .orElse(Boolean.FALSE);
     }
+
+    private static boolean isInProgressJobOutdated(V1Job job, DomainPresenceInfo info) {
+      return Optional.ofNullable(job)
+          .map(j -> !JobWatcher.isComplete(j) && hasIntrospectVersionChanged(j, info))
+          .orElse(false);
+    }
+
+    private static boolean hasIntrospectVersionChanged(@Nonnull V1Job job, DomainPresenceInfo info) {
+      return !Objects.equals(getIntrospectVersionLabelFromJob(job),
+          getIntrospectVersion(info));
+    }
+
+    private static String getIntrospectVersionLabelFromJob(V1Job job) {
+      return Optional.ofNullable(job)
+          .map(V1Job::getMetadata)
+          .map(V1ObjectMeta::getLabels)
+          .map(m -> m.get(INTROSPECTION_STATE_LABEL))
+          .orElse("");
+    }
+
   }
 
   static ReadDomainIntrospectorPodLogStep readDomainIntrospectorPodLog(Step next) {

operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java

@@ -371,6 +371,10 @@ private String getAsName() {
     return domainTopology.getAdminServerName();
   }
 
+  private String getIntrospectVersionLabel() {
+    return Optional.ofNullable(getDomain().getIntrospectVersion()).orElse(null);
+  }
+
   private long getIntrospectorJobActiveDeadlineSeconds(TuningParameters.PodTuning podTuning) {
     return Optional.ofNullable(getDomain().getIntrospectorJobActiveDeadlineSeconds())
         .orElse(podTuning.introspectorJobActiveDeadlineSeconds);
@@ -397,6 +401,7 @@ private V1ObjectMeta createMetadata() {
         new V1ObjectMeta()
           .name(getJobName())
           .namespace(getNamespace())
+          .putLabelsItem(LabelConstants.INTROSPECTION_STATE_LABEL, getIntrospectVersionLabel())
           .putLabelsItem(LabelConstants.DOMAINUID_LABEL, getDomainUid())
           .putLabelsItem(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"));
   }

operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java

@@ -274,6 +274,7 @@ public class DomainSpec extends BaseConfiguration {
       + "server must be restarted because of changes to any of the fields listed here: "
       + "https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/"
       + "domain-lifecycle/startup/#properties-that-cause-servers-to-be-restarted. "
+      + "The introspectVersion value must be a valid label value in Kubernetes. "
       + "See also `domains.spec.configuration.overrideDistributionStrategy`.")
   private String introspectVersion;
 

operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java

@@ -72,6 +72,7 @@
 import static oracle.kubernetes.operator.DomainProcessorTestSetup.NS;
 import static oracle.kubernetes.operator.DomainProcessorTestSetup.UID;
 import static oracle.kubernetes.operator.DomainUpPlanTest.StepChainMatcher.hasChainWithStepsInOrder;
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTOR_JOB;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY;
 import static oracle.kubernetes.operator.ProcessingConstants.EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG;
@@ -780,6 +781,75 @@ void whenJobHasImagePullBackOffError_correctStepsExecuted() {
             "DomainIntrospectorJobStep"));
   }
 
+  @Test
+  void whenJobInProgressAndIntrospectVersionAdded_correctStepsExecuted() {
+    List<Step> nextSteps = new ArrayList<>();
+    V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID))
+        .status(new V1JobStatus());
+    testSupport.defineResources(job);
+    getConfigurator().withIntrospectVersion("v2");
+    IntrospectionTestUtils.defineResources(testSupport, "passed");
+    testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
+
+    JobHelper.ReplaceOrCreateStep.createNextSteps(nextSteps, testSupport.getPacket(), job, terminalStep);
+
+    assertThat(nextSteps.get(0), hasChainWithStepsInOrder("DeleteDomainIntrospectorJobStep",
+        "DomainIntrospectorJobStep"));
+  }
+
+  @Test
+  void whenJobInProgressAndIntrospectVersionChanged_correctStepsExecuted() {
+    List<Step> nextSteps = new ArrayList<>();
+    V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID))
+        .status(new V1JobStatus());
+    job.getMetadata().putLabelsItem(INTROSPECTION_STATE_LABEL, "v1");
+    testSupport.defineResources(job);
+    getConfigurator().withIntrospectVersion("v2");
+    IntrospectionTestUtils.defineResources(testSupport, "passed");
+    testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
+
+    JobHelper.ReplaceOrCreateStep.createNextSteps(nextSteps, testSupport.getPacket(), job, terminalStep);
+
+    assertThat(nextSteps.get(0), hasChainWithStepsInOrder("DeleteDomainIntrospectorJobStep",
+        "DomainIntrospectorJobStep"));
+  }
+
+  @Test
+  void whenJobInProgressAndIntrospectVersionUnchanged_correctStepsExecuted() {
+    List<Step> nextSteps = new ArrayList<>();
+    V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID))
+        .status(new V1JobStatus());
+    job.getMetadata().putLabelsItem(INTROSPECTION_STATE_LABEL, "v2");
+    testSupport.defineResources(job);
+    getConfigurator().withIntrospectVersion("v2");
+    IntrospectionTestUtils.defineResources(testSupport, "passed");
+    testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
+
+    JobHelper.ReplaceOrCreateStep.createNextSteps(nextSteps, testSupport.getPacket(), job, terminalStep);
+
+    assertThat(nextSteps.get(0), hasChainWithStepsInOrder("WatchDomainIntrospectorJobReadyStep",
+        "ReadDomainIntrospectorPodStep", "ReadDomainIntrospectorPodLogStep",
+        "DeleteDomainIntrospectorJobStep", "IntrospectionConfigMapStep", "TerminalStep"));
+  }
+
+  @Test
+  void whenJobInProgressAndNullIntrospectVersionUnchanged_correctStepsExecuted() {
+    List<Step> nextSteps = new ArrayList<>();
+    V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID))
+        .status(new V1JobStatus());
+    job.getMetadata().putLabelsItem(INTROSPECTION_STATE_LABEL, null);
+    testSupport.defineResources(job);
+    getConfigurator().withIntrospectVersion(null);
+    IntrospectionTestUtils.defineResources(testSupport, "passed");
+    testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
+
+    JobHelper.ReplaceOrCreateStep.createNextSteps(nextSteps, testSupport.getPacket(), job, terminalStep);
+
+    assertThat(nextSteps.get(0), hasChainWithStepsInOrder("WatchDomainIntrospectorJobReadyStep",
+        "ReadDomainIntrospectorPodStep", "ReadDomainIntrospectorPodLogStep",
+        "DeleteDomainIntrospectorJobStep", "IntrospectionConfigMapStep", "TerminalStep"));
+  }
+
   @Test
   void whenCurrentJobIsNull_correctStepsExecuted() {
     List<Step> nextSteps = new ArrayList<>();

operator/src/test/java/oracle/kubernetes/operator/helpers/JobHelperTest.java

@@ -341,6 +341,14 @@ private V1JobSpec createJobSpec() {
     return domainIntrospectorJobStepContext.createJobSpec(TuningParameters.getInstance());
   }
 
+  private V1Job createJob() {
+    Packet packet = new Packet();
+    packet
+        .getComponents()
+        .put(ProcessingConstants.DOMAIN_COMPONENT_NAME, Component.createFor(domainPresenceInfo));
+    return new DomainIntrospectorJobStepContext(packet).getJobModel();
+  }
+
   @Test
   void introspectorPodStartsWithDefaultUser_Mem_Args_environmentVariable() {
     V1JobSpec jobSpec = createJobSpec();
@@ -1237,6 +1245,24 @@ void whenDomainIsIstioEnabled_istioLocalhostBindingsDisabledEnv_hasIstioUseLocal
     );
   }
 
+  @Test
+  void whenDomainHasIntrospectVersion_jobMetatadataCreatedWithLabel() {
+    final String INTROSPECT_VERSION = "v123";
+    configureDomain().withIntrospectVersion(INTROSPECT_VERSION);
+
+    V1Job job = createJob();
+    assertThat(job.getMetadata().getLabels().get(LabelConstants.INTROSPECTION_STATE_LABEL),
+        is(INTROSPECT_VERSION));
+  }
+
+  @Test
+  void whenDomainHasNoIntrospectVersion_jobMetatadataCreatedWithoutNoLabel() {
+    configureDomain().withIntrospectVersion(null);
+
+    V1Job job = createJob();
+    assertThat(job.getMetadata().getLabels().get(LabelConstants.INTROSPECTION_STATE_LABEL), is(nullValue()));
+  }
+
   private void markJobCompleted(V1Job job) {
     job.setStatus(new V1JobStatus().addConditionsItem(new V1JobCondition().status("True").type("Complete")));
   }