Skip to content

Commit 35de126

Browse files
ankediaankedia
authored
OWLS-90180 Optimize detection of when to run introspection (#2430)
* OWLS-90180 - Optimize introspector job run detection.
1 parent 3b44bdc commit 35de126

File tree

6 files changed

+168
-50
lines changed

6 files changed

+168
-50
lines changed

‎operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -303,15 +303,14 @@ private static Step domainIntrospectionSteps(DomainPresenceInfo info) {
303303
return Step.chain(
304304
ConfigMapHelper.readIntrospectionVersionStep(info.getNamespace(), info.getDomainUid()),
305305
new IntrospectionRequestStep(info),
306-
JobHelper.deleteDomainIntrospectorJobStep(null),
307-
JobHelper.createDomainIntrospectorJobStep(null));
306+
JobHelper.replaceOrCreateDomainIntrospectorJobStep(null));
308307
}
309308

310-
private static class IntrospectionRequestStep extends Step {
309+
public static class IntrospectionRequestStep extends Step {
311310

312311
private final String requestedIntrospectVersion;
313312

314-
IntrospectionRequestStep(DomainPresenceInfo info) {
313+
public IntrospectionRequestStep(DomainPresenceInfo info) {
315314
this.requestedIntrospectVersion = info.getDomain().getIntrospectVersion();
316315
}
317316

‎operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public interface LabelConstants {
2222
String MODEL_IN_IMAGE_DOMAINZIP_HASH = "weblogic.modelInImageDomainZipHash";
2323
String INTROSPECTION_STATE_LABEL = "weblogic.introspectVersion";
2424
String MII_UPDATED_RESTART_REQUIRED_LABEL = "weblogic.configChangesPendingRestart";
25+
String INTROSPECTION_DOMAIN_SPEC_GENERATION = "weblogic.domainSpecGeneration";
2526

2627
static String forDomainUidSelector(String uid) {
2728
return String.format("%s=%s", DOMAINUID_LABEL, uid);

‎operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.io.BufferedReader;
77
import java.io.IOException;
88
import java.io.StringReader;
9+
import java.time.OffsetDateTime;
910
import java.util.ArrayList;
1011
import java.util.Collection;
1112
import java.util.Collections;
@@ -49,6 +50,7 @@
4950
import org.yaml.snakeyaml.Yaml;
5051

5152
import static java.lang.System.lineSeparator;
53+
import static java.time.temporal.ChronoUnit.MILLIS;
5254
import static oracle.kubernetes.operator.DomainStatusUpdater.BAD_TOPOLOGY;
5355
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAINZIP_HASH;
5456
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH;
@@ -57,6 +59,7 @@
5759
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5;
5860
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SIT_CONFIG_FILE_PREFIX;
5961
import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME;
62+
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
6063
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
6164
import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_VALIDATION_ERRORS;
6265
import static oracle.kubernetes.operator.helpers.KubernetesUtils.getDomainUidLabel;
@@ -319,8 +322,11 @@ class ReadResponseStep extends DefaultResponseStep<V1ConfigMap> {
319322

320323
@Override
321324
public NextAction onSuccess(Packet packet, CallResponse<V1ConfigMap> callResponse) {
322-
DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getDomain).map(Domain::getIntrospectVersion)
325+
Domain domain = DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getDomain).orElse(null);
326+
Optional.ofNullable(domain).map(Domain::getIntrospectVersion)
323327
.ifPresent(value -> addLabel(INTROSPECTION_STATE_LABEL, value));
328+
Optional.ofNullable(domain).map(Domain::getMetadata).map(V1ObjectMeta::getGeneration)
329+
.ifPresent(value -> addLabel(INTROSPECTION_DOMAIN_SPEC_GENERATION, value.toString()));
324330
V1ConfigMap existingMap = withoutTransientData(callResponse.getResult());
325331
if (existingMap == null) {
326332
return doNext(createConfigMap(getNext()), packet);
@@ -504,9 +510,8 @@ public NextAction apply(Packet packet) {
504510
}
505511

506512
private long timeSinceJobStart(Packet packet) {
507-
return System.currentTimeMillis() - ((Long) packet.get(JobHelper.START_TIME));
513+
return ((OffsetDateTime)packet.get(JobHelper.START_TIME)).until(OffsetDateTime.now(), MILLIS);
508514
}
509-
510515
}
511516

512517
static class IntrospectionLoader {
@@ -870,12 +875,26 @@ public NextAction onSuccess(Packet packet, CallResponse<V1ConfigMap> callRespons
870875

871876
if (domainTopology != null) {
872877
recordTopology(packet, packet.getSpi(DomainPresenceInfo.class), domainTopology);
878+
recordIntrospectVersionAndGeneration(result, packet);
873879
return doNext(DomainValidationSteps.createValidateDomainTopologyStep(getNext()), packet);
874880
} else {
875881
return doNext(packet);
876882
}
877883
}
878884

885+
private void recordIntrospectVersionAndGeneration(V1ConfigMap result, Packet packet) {
886+
Map<String, String> labels = Optional.ofNullable(result)
887+
.map(V1ConfigMap::getMetadata)
888+
.map(V1ObjectMeta::getLabels).orElse(null);
889+
890+
Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_STATE_LABEL))
891+
.ifPresentOrElse(
892+
version -> packet.put(INTROSPECTION_STATE_LABEL, version),
893+
() -> packet.remove(INTROSPECTION_STATE_LABEL));
894+
Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
895+
.ifPresent(generation -> packet.put(INTROSPECTION_DOMAIN_SPEC_GENERATION, generation));
896+
}
897+
879898
private String getTopologyYaml(Map<String, String> data) {
880899
return data.get(IntrospectorConfigMapConstants.TOPOLOGY_YAML);
881900
}

‎operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java‎

Lines changed: 136 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package oracle.kubernetes.operator.helpers;
55

6+
import java.time.OffsetDateTime;
67
import java.util.ArrayList;
78
import java.util.Collections;
89
import java.util.List;
@@ -21,6 +22,7 @@
2122
import io.kubernetes.client.openapi.models.V1PodSpec;
2223
import io.kubernetes.client.openapi.models.V1Volume;
2324
import io.kubernetes.client.openapi.models.V1VolumeMount;
25+
import oracle.kubernetes.operator.DomainProcessorImpl;
2426
import oracle.kubernetes.operator.DomainStatusUpdater;
2527
import oracle.kubernetes.operator.IntrospectorConfigMapConstants;
2628
import oracle.kubernetes.operator.JobWatcher;
@@ -49,9 +51,12 @@
4951
import oracle.kubernetes.weblogic.domain.model.ManagedServer;
5052
import oracle.kubernetes.weblogic.domain.model.ServerEnvVars;
5153

54+
import static java.time.temporal.ChronoUnit.SECONDS;
5255
import static oracle.kubernetes.operator.DomainSourceType.FromModel;
5356
import static oracle.kubernetes.operator.DomainStatusUpdater.INSPECTING_DOMAIN_PROGRESS_REASON;
5457
import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStartedEventStep;
58+
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
59+
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
5560
import static oracle.kubernetes.operator.logging.MessageKeys.INTROSPECTOR_JOB_FAILED;
5661
import static oracle.kubernetes.operator.logging.MessageKeys.INTROSPECTOR_JOB_FAILED_DETAIL;
5762

@@ -90,18 +95,40 @@ private static boolean runIntrospector(Packet packet, DomainPresenceInfo info) {
9095
LOGGER.fine("runIntrospector topology: " + topology);
9196
LOGGER.fine("runningServersCount: " + runningServersCount(info));
9297
LOGGER.fine("creatingServers: " + creatingServers(info));
98+
LOGGER.fine("isModelInImageUpdate: " + isModelInImageUpdate(packet, info));
9399
return topology == null
94-
|| isBringingUpNewDomain(info)
95-
|| introspectionRequested(packet)
96-
|| isModelInImageUpdate(packet, info);
100+
|| isBringingUpNewDomain(packet, info)
101+
|| isIntrospectionRequestedAndRemove(packet)
102+
|| isModelInImageUpdate(packet, info)
103+
|| isIntrospectVersionChanged(packet, info);
97104
}
98105

99-
private static boolean isBringingUpNewDomain(DomainPresenceInfo info) {
100-
return runningServersCount(info) == 0 && creatingServers(info);
106+
private static boolean isBringingUpNewDomain(Packet packet, DomainPresenceInfo info) {
107+
return runningServersCount(info) == 0 && creatingServers(info) && isGenerationChanged(packet, info);
101108
}
102109

103-
private static boolean introspectionRequested(Packet packet) {
104-
return packet.containsKey(ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED);
110+
private static boolean isIntrospectionRequestedAndRemove(Packet packet) {
111+
return packet.remove(ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED) != null;
112+
}
113+
114+
private static boolean isIntrospectVersionChanged(Packet packet, DomainPresenceInfo info) {
115+
return Optional.ofNullable(packet.get(INTROSPECTION_STATE_LABEL))
116+
.map(introspectVersionLabel -> !introspectVersionLabel.equals(getIntrospectVersion(info))).orElse(false);
117+
}
118+
119+
private static boolean isGenerationChanged(Packet packet, DomainPresenceInfo info) {
120+
return Optional.ofNullable(packet.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
121+
.map(gen -> !gen.equals(getGeneration(info))).orElse(true);
122+
}
123+
124+
private static String getIntrospectVersion(DomainPresenceInfo info) {
125+
return Optional.ofNullable(info.getDomain()).map(Domain::getSpec).map(s -> s.getIntrospectVersion())
126+
.orElse("");
127+
}
128+
129+
private static String getGeneration(DomainPresenceInfo info) {
130+
return Optional.ofNullable(info.getDomain()).map(Domain::getMetadata).map(m -> m.getGeneration().toString())
131+
.orElse("");
105132
}
106133

107134
private static boolean isModelInImageUpdate(Packet packet, DomainPresenceInfo info) {
@@ -178,13 +205,13 @@ static boolean creatingServers(DomainPresenceInfo info) {
178205
}
179206

180207
/**
181-
* Factory for {@link Step} that deletes WebLogic domain introspector job.
208+
* Factory for {@link Step} that replaces or creates WebLogic domain introspector job.
182209
*
183210
* @param next Next processing step
184-
* @return Step for deleting the domain introsepctor jod
211+
* @return Step for replacing or creating the domain introsepctor jod
185212
*/
186-
public static Step deleteDomainIntrospectorJobStep(Step next) {
187-
return new DeleteIntrospectorJobStep(next);
213+
public static Step replaceOrCreateDomainIntrospectorJobStep(Step next) {
214+
return new ReplaceOrCreateIntrospectorJobStep(next);
188215
}
189216

190217
private static Step createWatchDomainIntrospectorJobReadyStep(Step next) {
@@ -374,7 +401,7 @@ public NextAction apply(Packet packet) {
374401
if (runIntrospector(packet, info)) {
375402
JobStepContext context = new DomainIntrospectorJobStepContext(packet);
376403

377-
packet.putIfAbsent(START_TIME, System.currentTimeMillis());
404+
packet.putIfAbsent(START_TIME, OffsetDateTime.now());
378405

379406
return doNext(
380407
Step.chain(
@@ -392,50 +419,60 @@ public NextAction apply(Packet packet) {
392419
}
393420
}
394421

395-
private static class DeleteIntrospectorJobStep extends Step {
422+
private static class ReplaceOrCreateIntrospectorJobStep extends Step {
396423

397424
static final int JOB_DELETE_TIMEOUT_SECONDS = 1;
398425

399-
DeleteIntrospectorJobStep(Step next) {
426+
ReplaceOrCreateIntrospectorJobStep(Step next) {
400427
super(next);
401428
}
402429

403430
@Override
404431
public NextAction apply(Packet packet) {
405-
return doNext(deleteJob(packet, getNext()), packet);
432+
return doNext(replaceOrCreateJob(packet, getNext()), packet);
406433
}
407434

408-
String getJobDeletedMessageKey() {
409-
return MessageKeys.JOB_DELETED;
435+
436+
private Step replaceOrCreateJob(Packet packet, Step next) {
437+
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
438+
return new CallBuilder().readJobAsync(JobHelper.createJobName(info.getDomain().getDomainUid()),
439+
info.getNamespace(), info.getDomain().getDomainUid(),
440+
new ReplaceOrCreateStep(next));
410441
}
411442

412-
void logJobDeleted(String domainUid, String namespace, String jobName, Packet packet) {
413-
V1Job domainIntrospectorJob =
414-
(V1Job) packet.remove(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB);
443+
private class ReplaceOrCreateStep extends DefaultResponseStep {
415444

416-
packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
417-
if (domainIntrospectorJob != null
418-
&& !JobWatcher.isComplete(domainIntrospectorJob)) {
419-
logIntrospectorFailure(packet, domainIntrospectorJob);
445+
ReplaceOrCreateStep(Step next) {
446+
super(next);
420447
}
421-
packet.remove(ProcessingConstants.JOB_POD_NAME);
422448

423-
LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
424-
}
449+
@Override
450+
public NextAction onSuccess(Packet packet, CallResponse callResponse) {
451+
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
452+
String namespace = info.getNamespace();
453+
V1Job job = (V1Job) callResponse.getResult();
454+
if ((job != null) && (packet.get(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB) == null)) {
455+
packet.put(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB, job);
456+
}
425457

426-
private Step deleteJob(Packet packet, Step next) {
427-
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
428-
java.lang.String domainUid = info.getDomain().getDomainUid();
429-
java.lang.String namespace = info.getNamespace();
430-
String jobName = JobHelper.createJobName(domainUid);
431-
logJobDeleted(domainUid, namespace, jobName, packet);
432-
return new CallBuilder().withTimeoutSeconds(JOB_DELETE_TIMEOUT_SECONDS)
433-
.deleteJobAsync(
434-
jobName,
435-
namespace,
436-
domainUid,
437-
new V1DeleteOptions().propagationPolicy("Foreground"),
438-
new DefaultResponseStep<>(next));
458+
if (job != null) {
459+
packet.putIfAbsent(START_TIME, Optional.ofNullable(job.getMetadata())
460+
.map(m -> m.getCreationTimestamp()).orElse(OffsetDateTime.now()));
461+
return doNext(Step.chain(
462+
createProgressingStartedEventStep(info, INSPECTING_DOMAIN_PROGRESS_REASON, true, null),
463+
readDomainIntrospectorPodLogStep(null),
464+
deleteDomainIntrospectorJobStep(null),
465+
ConfigMapHelper.createIntrospectorConfigMapStep(null),
466+
ConfigMapHelper.readExistingIntrospectorConfigMap(namespace, info.getDomainUid()),
467+
new DomainProcessorImpl.IntrospectionRequestStep(info),
468+
createDomainIntrospectorJobStep(getNext())), packet);
469+
} else {
470+
packet.putIfAbsent(START_TIME, OffsetDateTime.now());
471+
return doNext(Step.chain(
472+
ConfigMapHelper.readExistingIntrospectorConfigMap(namespace, info.getDomainUid()),
473+
createDomainIntrospectorJobStep(getNext())), packet);
474+
}
475+
}
439476
}
440477
}
441478

@@ -507,17 +544,32 @@ public NextAction onSuccess(Packet packet, CallResponse<String> callResponse) {
507544
jobConditionsReason.add(DomainStatusUpdater.ERR_INTROSPECTOR);
508545
}
509546
//Introspector job is incomplete, update domain status and terminate processing
547+
Step nextStep = null;
548+
int retryIntervalSeconds = TuningParameters.getInstance().getMainTuning().domainPresenceRecheckIntervalSeconds;
549+
550+
if (OffsetDateTime.now().isAfter(
551+
getJobCreationTime(domainIntrospectorJob).plus(retryIntervalSeconds, SECONDS))) {
552+
//Introspector job is incomplete and current time is greater than the lazy deletion time for the job,
553+
//update the domain status and execute the next step
554+
nextStep = getNext();
555+
}
556+
510557
return doNext(
511558
DomainStatusUpdater.createFailureRelatedSteps(
512559
onSeparateLines(jobConditionsReason),
513560
onSeparateLines(severeStatuses),
514-
null),
561+
nextStep),
515562
packet);
516563
}
517564

518565
return doNext(packet);
519566
}
520567

568+
private OffsetDateTime getJobCreationTime(V1Job domainIntrospectorJob) {
569+
return Optional.ofNullable(domainIntrospectorJob.getMetadata())
570+
.map(m -> m.getCreationTimestamp()).orElse(OffsetDateTime.now());
571+
}
572+
521573
private boolean isNotComplete(V1Job domainIntrospectorJob) {
522574
return !JobWatcher.isComplete(domainIntrospectorJob);
523575
}
@@ -611,6 +663,49 @@ private static void logIntrospectorFailure(Packet packet, V1Job domainIntrospect
611663
}
612664
}
613665

666+
static class DeleteDomainIntrospectorJobStep extends Step {
667+
668+
DeleteDomainIntrospectorJobStep(Step next) {
669+
super(next);
670+
}
671+
672+
@Override
673+
public NextAction apply(Packet packet) {
674+
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
675+
String jobName = JobHelper.createJobName(info.getDomainUid());
676+
logJobDeleted(info.getDomainUid(), info.getNamespace(), jobName, packet);
677+
return doNext(new CallBuilder().withTimeoutSeconds(ReplaceOrCreateIntrospectorJobStep.JOB_DELETE_TIMEOUT_SECONDS)
678+
.deleteJobAsync(
679+
jobName,
680+
info.getNamespace(),
681+
info.getDomainUid(),
682+
new V1DeleteOptions().propagationPolicy("Foreground"),
683+
new DefaultResponseStep<>(getNext())), packet);
684+
}
685+
}
686+
687+
public static Step deleteDomainIntrospectorJobStep(Step next) {
688+
return new DeleteDomainIntrospectorJobStep(next);
689+
}
690+
691+
static void logJobDeleted(String domainUid, String namespace, String jobName, Packet packet) {
692+
V1Job domainIntrospectorJob =
693+
(V1Job) packet.remove(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB);
694+
695+
packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
696+
if (domainIntrospectorJob != null
697+
&& !JobWatcher.isComplete(domainIntrospectorJob)) {
698+
logIntrospectorFailure(packet, domainIntrospectorJob);
699+
}
700+
packet.remove(ProcessingConstants.JOB_POD_NAME);
701+
702+
LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
703+
}
704+
705+
static String getJobDeletedMessageKey() {
706+
return MessageKeys.JOB_DELETED;
707+
}
708+
614709
private static class ReadDomainIntrospectorPodStep extends Step {
615710

616711
ReadDomainIntrospectorPodStep(Step next) {

‎operator/src/test/java/oracle/kubernetes/operator/DomainUpPlanTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ public void useSequenceBeforeAdminServerStep() {
137137
plan,
138138
hasChainWithStepsInOrder(
139139
"DomainPresenceStep",
140-
"DomainIntrospectorJobStep",
140+
"ReplaceOrCreateIntrospectorJobStep",
141141
"BeforeAdminServiceStep",
142142
"AdminPodStep",
143143
"ForServerStep",

0 commit comments

Comments
 (0)