Skip to content

Commit 30212cc

Browse files
robertpatrickrobertpatrick
authored
Merge pull request #2681 from oracle/owls94649_fix_backport_3.3
2 parents 01a0634 + 4192696 commit 30212cc

File tree

6 files changed

+196
-10
lines changed

6 files changed

+196
-10
lines changed

‎operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import io.kubernetes.client.openapi.models.CoreV1Event;
2424
import io.kubernetes.client.openapi.models.V1ConfigMap;
2525
import io.kubernetes.client.openapi.models.V1ContainerState;
26+
import io.kubernetes.client.openapi.models.V1ContainerStateWaiting;
2627
import io.kubernetes.client.openapi.models.V1ContainerStatus;
2728
import io.kubernetes.client.openapi.models.V1ObjectMeta;
2829
import io.kubernetes.client.openapi.models.V1ObjectReference;
@@ -1380,6 +1381,21 @@ private void invoke() {
13801381
DomainStatusUpdater.createFailureRelatedSteps(
13811382
info, waiting.getReason(), waiting.getMessage(), null)));
13821383
break;
1384+
case INIT_CONTAINERS_NOT_READY:
1385+
List<String> waitingReasons = new ArrayList<>();
1386+
List<String> waitingMessages = new ArrayList<>();
1387+
1388+
Optional.ofNullable(getInitContainerStatuses(introspectorPod))
1389+
.orElseGet(Collections::emptyList).stream()
1390+
.forEach(status -> {
1391+
waitingMessages.add(getWaitingMessageFromStatus(status));
1392+
waitingReasons.add(getWaitingReason(status));
1393+
});
1394+
if (!waitingReasons.isEmpty()) {
1395+
delegate.runSteps(DomainStatusUpdater.createFailureRelatedSteps(
1396+
info, onSeparateLines(waitingReasons), onSeparateLines(waitingMessages), null));
1397+
}
1398+
break;
13831399
case TERMINATED_ERROR_REASON:
13841400
Optional.ofNullable(getMatchingContainerStatus())
13851401
.map(V1ContainerStatus::getState)
@@ -1408,6 +1424,30 @@ private void invoke() {
14081424
}
14091425
}
14101426

1427+
private String getWaitingReason(V1ContainerStatus status) {
1428+
return Optional.ofNullable(status)
1429+
.map(V1ContainerStatus::getState)
1430+
.map(V1ContainerState::getWaiting)
1431+
.map(V1ContainerStateWaiting::getReason)
1432+
.orElse(null);
1433+
}
1434+
1435+
private String getWaitingMessageFromStatus(V1ContainerStatus status) {
1436+
return Optional.ofNullable(status)
1437+
.map(V1ContainerStatus::getState)
1438+
.map(V1ContainerState::getWaiting)
1439+
.map(V1ContainerStateWaiting::getMessage)
1440+
.orElse(null);
1441+
}
1442+
1443+
private List<V1ContainerStatus> getInitContainerStatuses(V1Pod pod) {
1444+
return Optional.ofNullable(pod.getStatus()).map(V1PodStatus::getInitContainerStatuses).orElse(null);
1445+
}
1446+
1447+
private String onSeparateLines(List<String> waitingReasons) {
1448+
return String.join(System.lineSeparator(), waitingReasons);
1449+
}
1450+
14111451
private boolean isNotTerminatedByOperator() {
14121452
return notNullOrEmpty(getPodStatusReason()) || notNullOrEmpty(getPodStatusMessage()) || !isJobPodTerminated();
14131453
}

‎operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ public class PodWatcher extends Watcher<V1Pod> implements WatchListener<V1Pod>,
6060
public enum PodStatus {
6161
PHASE_FAILED,
6262
WAITING_NON_NULL_MESSAGE,
63+
INIT_CONTAINERS_NOT_READY,
6364
TERMINATED_ERROR_REASON,
6465
UNSCHEDULABLE,
6566
SUCCESS
@@ -214,6 +215,8 @@ static PodStatus getPodStatus(@Nonnull V1Pod pod) {
214215
return PodStatus.PHASE_FAILED;
215216
} else if (notReady(conStatus) && getContainerStateWaitingMessage(conStatus) != null) {
216217
return PodStatus.WAITING_NON_NULL_MESSAGE;
218+
} else if (initContainersNotReady(pod)) {
219+
return PodStatus.INIT_CONTAINERS_NOT_READY;
217220
} else if (notReady(conStatus) && getContainerStateTerminatedReason(conStatus).contains("Error")) {
218221
return PodStatus.TERMINATED_ERROR_REASON;
219222
} else if (isUnschedulable(pod)) {
@@ -222,6 +225,11 @@ static PodStatus getPodStatus(@Nonnull V1Pod pod) {
222225
return PodStatus.SUCCESS;
223226
}
224227

228+
private static boolean initContainersNotReady(@Nonnull V1Pod pod) {
229+
return notReady(Optional.ofNullable(pod.getStatus()).map(s -> s.getInitContainerStatuses())
230+
.orElseGet(Collections::emptyList));
231+
}
232+
225233
static V1ContainerStatus getContainerStatus(@Nonnull V1Pod pod) {
226234
return getContainerStatuses(pod)
227235
.stream()
@@ -257,6 +265,11 @@ private static String getReason(V1PodCondition podCondition) {
257265
return Optional.ofNullable(podCondition).map(V1PodCondition::getReason).orElse("");
258266
}
259267

268+
private static boolean notReady(List<V1ContainerStatus> initContainerStatuses) {
269+
return Optional.ofNullable(initContainerStatuses)
270+
.orElseGet(Collections::emptyList).stream().anyMatch(status -> notReady(status));
271+
}
272+
260273
private static boolean notReady(V1ContainerStatus conStatus) {
261274
return !Optional.ofNullable(conStatus).map(V1ContainerStatus::getReady).orElse(false);
262275
}

‎operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public interface ProcessingConstants {
2828
String DOMAIN_TOPOLOGY = "domainTopology";
2929
String JOB_POD_NAME = "jobPodName";
3030
String JOB_POD_CONTAINER_WAITING_REASON = "jobPodContainerWaitingReason";
31+
String JOB_POD_INIT_CONTAINER_WAITING_REASON = "jobPodInitContainerWaitingReason";
3132
String DOMAIN_INTROSPECTOR_JOB = "domainIntrospectorJob";
3233
String DOMAIN_INTROSPECTOR_LOG_RESULT = "domainIntrospectorLogResult";
3334
String DOMAIN_INTROSPECT_REQUESTED = "domainIntrospectRequested";

‎operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,10 @@ private static boolean isImagePullError(String jobPodContainerWaitingReason) {
496496
.orElse(false);
497497
}
498498

499+
private List<V1ContainerStatus> getInitContainerStatuses(V1Pod pod) {
500+
return Optional.ofNullable(pod.getStatus()).map(V1PodStatus::getInitContainerStatuses).orElse(null);
501+
}
502+
499503
private static boolean isJobTimedout(DomainPresenceInfo info) {
500504
return Objects.equals(getReason(info), "DeadlineExceeded") || getMessage(info).contains("DeadlineExceeded");
501505
}
@@ -564,22 +568,37 @@ private void recordJobPodNameAndStatus(Packet packet, V1Pod pod) {
564568
.map(V1PodStatus::getContainerStatuses).map(statuses -> statuses.get(0))
565569
.map(V1ContainerStatus::getState).map(V1ContainerState::getWaiting)
566570
.map(V1ContainerStateWaiting::getReason).orElse(null));
571+
packet.put(ProcessingConstants.JOB_POD_INIT_CONTAINER_WAITING_REASON, getInitContainerWaitingMessages(pod));
567572
}
568573
}
574+
575+
private Boolean getInitContainerWaitingMessages(V1Pod pod) {
576+
return Optional.ofNullable(getInitContainerStatuses(pod)).orElseGet(Collections::emptyList).stream()
577+
.anyMatch(status -> isImagePullError(getWaitingReason(status)));
578+
}
579+
580+
private String getWaitingReason(V1ContainerStatus status) {
581+
return Optional.ofNullable(status)
582+
.map(V1ContainerStatus::getState)
583+
.map(V1ContainerState::getWaiting)
584+
.map(V1ContainerStateWaiting::getReason)
585+
.orElse(null);
586+
}
569587
}
570588

571589
static OffsetDateTime createNextSteps(List<Step> nextSteps, Packet packet, V1Job job, Step next) {
572590
OffsetDateTime jobStartTime;
573591
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
574592
String namespace = info.getNamespace();
575-
String jobPodContainerWaitingReason = (String) packet.get(ProcessingConstants.JOB_POD_CONTAINER_WAITING_REASON);
593+
String jobPodContainerWaitingReason = packet.getValue(ProcessingConstants.JOB_POD_CONTAINER_WAITING_REASON);
594+
Boolean jobInitContainerImagePullError = getjobInitContainerImagePullError(packet);
576595

577596
if (job != null) {
578597
jobStartTime = Optional.ofNullable(job.getMetadata())
579598
.map(V1ObjectMeta::getCreationTimestamp).orElse(OffsetDateTime.now());
580599
String lastIntrospectJobProcessedId = getLastIntrospectJobProcessedId(info);
581600

582-
if (isJobTimedout(info) || (isImagePullError(jobPodContainerWaitingReason))) {
601+
if (isJobTimedout(info) || (isImagePullError(jobPodContainerWaitingReason)) || jobInitContainerImagePullError) {
583602
jobStartTime = OffsetDateTime.now();
584603
packet.put(DOMAIN_INTROSPECT_REQUESTED, ReadDomainIntrospectorPodLogResponseStep.INTROSPECTION_FAILED);
585604
nextSteps.add(Step.chain(deleteDomainIntrospectorJobStep(null),
@@ -602,6 +621,11 @@ static OffsetDateTime createNextSteps(List<Step> nextSteps, Packet packet, V1Job
602621
}
603622
return jobStartTime;
604623
}
624+
625+
private static Boolean getjobInitContainerImagePullError(Packet packet) {
626+
return Optional.ofNullable(packet.<Boolean>getValue(ProcessingConstants.JOB_POD_INIT_CONTAINER_WAITING_REASON))
627+
.orElse(Boolean.FALSE);
628+
}
605629
}
606630

607631
static ReadDomainIntrospectorPodLogStep readDomainIntrospectorPodLog(Step next) {

‎operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,16 @@
2626
import com.meterware.simplestub.StaticStubSupport;
2727
import io.kubernetes.client.openapi.models.CoreV1Event;
2828
import io.kubernetes.client.openapi.models.V1ConfigMap;
29+
import io.kubernetes.client.openapi.models.V1ContainerState;
30+
import io.kubernetes.client.openapi.models.V1ContainerStateWaiting;
31+
import io.kubernetes.client.openapi.models.V1ContainerStatus;
2932
import io.kubernetes.client.openapi.models.V1Job;
3033
import io.kubernetes.client.openapi.models.V1JobCondition;
3134
import io.kubernetes.client.openapi.models.V1JobStatus;
3235
import io.kubernetes.client.openapi.models.V1LabelSelector;
3336
import io.kubernetes.client.openapi.models.V1ObjectMeta;
3437
import io.kubernetes.client.openapi.models.V1Pod;
38+
import io.kubernetes.client.openapi.models.V1PodStatus;
3539
import io.kubernetes.client.openapi.models.V1Secret;
3640
import io.kubernetes.client.openapi.models.V1Service;
3741
import io.kubernetes.client.openapi.models.V1ServicePort;
@@ -52,6 +56,7 @@
5256
import oracle.kubernetes.operator.helpers.ServiceHelper;
5357
import oracle.kubernetes.operator.helpers.TuningParametersStub;
5458
import oracle.kubernetes.operator.helpers.UnitTestHash;
59+
import oracle.kubernetes.operator.logging.MessageKeys;
5560
import oracle.kubernetes.operator.rest.ScanCacheStub;
5661
import oracle.kubernetes.operator.utils.InMemoryCertificates;
5762
import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig;
@@ -87,16 +92,19 @@
8792
import static oracle.kubernetes.operator.LabelConstants.DOMAINUID_LABEL;
8893
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
8994
import static oracle.kubernetes.operator.LabelConstants.SERVERNAME_LABEL;
95+
import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTOR_JOB;
9096
import static oracle.kubernetes.operator.WebLogicConstants.RUNNING_STATE;
9197
import static oracle.kubernetes.operator.WebLogicConstants.SHUTDOWN_STATE;
9298
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.CONFIG_MAP;
9399
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.DOMAIN;
100+
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.JOB;
94101
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.POD;
95102
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.SERVICE;
96103
import static oracle.kubernetes.operator.logging.MessageKeys.NOT_STARTING_DOMAINUID_THREAD;
97104
import static oracle.kubernetes.utils.LogMatcher.containsFine;
98105
import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_ALWAYS;
99106
import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_NEVER;
107+
import static oracle.kubernetes.weblogic.domain.model.DomainConditionType.Failed;
100108
import static org.hamcrest.Matchers.allOf;
101109
import static org.hamcrest.Matchers.contains;
102110
import static org.hamcrest.Matchers.empty;
@@ -120,6 +128,7 @@ class DomainProcessorTest {
120128
private static final String[] MANAGED_SERVER_NAMES =
121129
IntStream.rangeClosed(1, MAX_SERVERS).mapToObj(DomainProcessorTest::getManagedServerName).toArray(String[]::new);
122130
public static final String DOMAIN_NAME = "base_domain";
131+
private TestUtils.ConsoleHandlerMemento consoleHandlerMemento;
123132

124133
@Nonnull
125134
private static String getManagedServerName(int n) {
@@ -165,6 +174,9 @@ private static WlsDomainConfig createDomainConfig() {
165174

166175
@BeforeEach
167176
public void setUp() throws Exception {
177+
consoleHandlerMemento = TestUtils.silenceOperatorLogger()
178+
.collectLogMessages(logRecords, NOT_STARTING_DOMAINUID_THREAD).withLogLevel(Level.FINE);
179+
mementos.add(consoleHandlerMemento);
168180
mementos.add(TestUtils.silenceOperatorLogger()
169181
.collectLogMessages(logRecords, NOT_STARTING_DOMAINUID_THREAD).withLogLevel(Level.FINE));
170182
mementos.add(testSupport.install());
@@ -1337,4 +1349,71 @@ private void defineDuplicateServerNames() {
13371349
domain.getSpec().getManagedServers().add(new ManagedServer().withServerName("ms1"));
13381350
domain.getSpec().getManagedServers().add(new ManagedServer().withServerName("ms1"));
13391351
}
1352+
1353+
@Test
1354+
void whenIntrospectionJobInitContainerHasImagePullFailure_jobRecreatedAndFailedConditionCleared() throws Exception {
1355+
consoleHandlerMemento.ignoringLoggedExceptions(RuntimeException.class);
1356+
consoleHandlerMemento.ignoreMessage(MessageKeys.NOT_STARTING_DOMAINUID_THREAD);
1357+
jobStatus = createBackoffStatus();
1358+
establishPreviousIntrospection(null);
1359+
defineIntrospectionWithInitContainerImagePullError();
1360+
testSupport.doOnDelete(JOB, j -> deletePod());
1361+
testSupport.doOnCreate(JOB, j -> createJobPodAndSetCompletedStatus(job));
1362+
domainConfigurator.withIntrospectVersion(NEW_INTROSPECTION_STATE);
1363+
processor.createMakeRightOperation(new DomainPresenceInfo(newDomain)).interrupt().execute();
1364+
1365+
assertThat(isDomainConditionFailed(), is(false));
1366+
}
1367+
1368+
V1JobStatus createBackoffStatus() {
1369+
return new V1JobStatus().addConditionsItem(new V1JobCondition().status("True").type("Failed")
1370+
.reason("BackoffLimitExceeded"));
1371+
}
1372+
1373+
private void defineIntrospectionWithInitContainerImagePullError() {
1374+
V1Job job = asFailedJob(createIntrospectorJob("IMAGE_PULL_FAILURE_JOB"));
1375+
testSupport.defineResources(job);
1376+
testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, job);
1377+
setJobPodInitContainerStatusImagePullError();
1378+
}
1379+
1380+
private void setJobPodInitContainerStatusImagePullError() {
1381+
testSupport.<V1Pod>getResourceWithName(POD, getJobName()).status(new V1PodStatus().initContainerStatuses(
1382+
Arrays.asList(new V1ContainerStatus().state(new V1ContainerState().waiting(
1383+
new V1ContainerStateWaiting().reason("ImagePullBackOff").message("Back-off pulling image"))))));
1384+
}
1385+
1386+
private V1Job asFailedJob(V1Job job) {
1387+
job.setStatus(new V1JobStatus().addConditionsItem(new V1JobCondition().status("True").type("Failed")
1388+
.reason("BackoffLimitExceeded")));
1389+
return job;
1390+
}
1391+
1392+
private V1Job createIntrospectorJob(String uid) {
1393+
return new V1Job().metadata(createJobMetadata(uid)).status(new V1JobStatus());
1394+
}
1395+
1396+
private V1ObjectMeta createJobMetadata(String uid) {
1397+
return new V1ObjectMeta().name(getJobName()).namespace(NS).creationTimestamp(SystemClock.now()).uid(uid);
1398+
}
1399+
1400+
private static String getJobName() {
1401+
return LegalNames.toJobIntrospectorName(UID);
1402+
}
1403+
1404+
private void deletePod() {
1405+
testSupport.deleteResources(new V1Pod().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS)));
1406+
}
1407+
1408+
private void createJobPodAndSetCompletedStatus(V1Job job) {
1409+
Map<String, String> labels = new HashMap<>();
1410+
labels.put(LabelConstants.JOBNAME_LABEL, getJobName());
1411+
testSupport.defineResources(POD,
1412+
new V1Pod().metadata(new V1ObjectMeta().name(getJobName()).labels(labels).namespace(NS)));
1413+
job.setStatus(createCompletedStatus());
1414+
}
1415+
1416+
private boolean isDomainConditionFailed() {
1417+
return newDomain.getStatus().getConditions().stream().anyMatch(c -> c.getType() == Failed);
1418+
}
13401419
}

‎operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectionStatusTest.java‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class IntrospectionStatusTest {
5050
private static final String UNSCHEDULABLE = "Unschedulable";
5151
private static final String IMAGE_PULL_BACKOFF = "ImagePullBackoff";
5252
private static final String DEADLINE_EXCEEDED = "DeadlineExceeded";
53+
private static final int MESSAGE_LENGTH = 10;
5354
private final List<Memento> mementos = new ArrayList<>();
5455
private final KubernetesTestSupport testSupport = new KubernetesTestSupport();
5556
private final Map<String, Map<String, DomainPresenceInfo>> presenceInfoMap = new HashMap<>();
@@ -208,6 +209,21 @@ void whenNewIntrospectorJobPodStatusReasonNullAfterImagePullFailure_patchDomain(
208209
assertThat(updatedDomain.getStatus().getMessage(), emptyOrNullString());
209210
}
210211

212+
@Test
213+
void whenPodHasInitContainerImagePullErrorWaitingMessage_updateDomainStatus() {
214+
processor.dispatchPodWatch(
215+
WatchEvent.createAddedEvent(
216+
createIntrospectorJobPodWithInitContainerStatus(createWaitingState(IMAGE_PULL_FAILURE, MESSAGE)))
217+
.toWatchResponse());
218+
219+
assertThat(getDomain().getStatus().getReason(), equalTo("ErrImagePull"));
220+
assertThat(getDomain().getStatus().getMessage(), equalTo(MESSAGE));
221+
}
222+
223+
private Domain getDomain() {
224+
return testSupport.getResourceWithName(KubernetesTestSupport.DOMAIN, UID);
225+
}
226+
211227
private V1Pod createIntrospectorJobPod(V1ContainerState waitingState) {
212228
return createIntrospectorJobPod(UID)
213229
.status(
@@ -224,14 +240,27 @@ private V1Pod createIntrospectorJobPod(V1ContainerState waitingState) {
224240
@SuppressWarnings("SameParameterValue")
225241
private V1Pod createIntrospectorJobPod(String domainUid) {
226242
return AnnotationHelper.withSha256Hash(
227-
new V1Pod()
228-
.metadata(
229-
withIntrospectorJobLabels(
230-
new V1ObjectMeta()
231-
.name(toJobIntrospectorName(domainUid) + getPodSuffix())
232-
.namespace(NS),
233-
domainUid))
234-
.spec(new V1PodSpec()));
243+
new V1Pod()
244+
.metadata(
245+
withIntrospectorJobLabels(
246+
new V1ObjectMeta()
247+
.name(toJobIntrospectorName(domainUid) + getPodSuffix())
248+
.namespace(NS),
249+
domainUid))
250+
.spec(new V1PodSpec()));
251+
}
252+
253+
private V1Pod createIntrospectorJobPodWithInitContainerStatus(V1ContainerState waitingState) {
254+
return createIntrospectorJobPod(UID)
255+
.status(
256+
new V1PodStatusBuilder()
257+
.addNewInitContainerStatus()
258+
.withImage(IMAGE_NAME)
259+
.withName(toJobIntrospectorName(UID))
260+
.withReady(false)
261+
.withState(waitingState)
262+
.endInitContainerStatus()
263+
.build());
235264
}
236265

237266
private V1Pod createIntrospectorJobPodWithConditions(V1PodCondition condition) {

0 commit comments

Comments
 (0)