Skip to content

Commit db00391

Browse files
marinakogmarinakog
authored
Added OCI LoadBalancer test to OKE certification run (#2329)
* Added OCI LoadBalancer test to OKE certification run
1 parent 251bc29 commit db00391

File tree

7 files changed

+316
-5
lines changed

7 files changed

+316
-5
lines changed

‎integration-tests/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@
295295
**/ItOperatorUpgrade,
296296
**/ItOperatorRestart,
297297
**/ItPodsShutdownOption,
298+
**/ItOCILoadBalancer,
298299
**/ItManageNameSpace
299300
</includes-failsafe>
300301
</properties>

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// Copyright (c) 2021, Oracle and/or its affiliates.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
package oracle.weblogic.kubernetes;
5+
6+
import java.util.ArrayList;
7+
import java.util.HashMap;
8+
import java.util.List;
9+
import java.util.Map;
10+
11+
import io.kubernetes.client.openapi.models.V1LoadBalancerIngress;
12+
import io.kubernetes.client.openapi.models.V1Service;
13+
import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
14+
import oracle.weblogic.kubernetes.annotations.IntegrationTest;
15+
import oracle.weblogic.kubernetes.annotations.Namespaces;
16+
import oracle.weblogic.kubernetes.logging.LoggingFacade;
17+
import org.awaitility.core.ConditionFactory;
18+
import org.junit.jupiter.api.AfterAll;
19+
import org.junit.jupiter.api.BeforeAll;
20+
import org.junit.jupiter.api.DisplayName;
21+
import org.junit.jupiter.api.Test;
22+
23+
import static java.util.concurrent.TimeUnit.MINUTES;
24+
import static java.util.concurrent.TimeUnit.SECONDS;
25+
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
26+
import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE;
27+
import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME;
28+
import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG;
29+
import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService;
30+
import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify;
31+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret;
32+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOCILoadBalancer;
33+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator;
34+
import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndCheckForServerNameInResponse;
35+
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
36+
import static org.assertj.core.api.Assertions.assertThat;
37+
import static org.awaitility.Awaitility.with;
38+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
39+
import static org.junit.jupiter.api.Assertions.assertNotNull;
40+
41+
/**
42+
* Verify OCI Load Balancer is installed and running.
43+
* Verify sample-war web application be accessed via OCI LoadBalancer.
44+
* Verify Load Balancing between two managed servers in the cluster
45+
*/
46+
@DisplayName("Verify the sample-app app can be accessed from "
47+
+ "all managed servers in the domain through OCI Load Balancer")
48+
@IntegrationTest
49+
class ItOCILoadBalancer {
50+
// domain constants
51+
private static final int replicaCount = 2;
52+
private static int managedServersCount = 2;
53+
private static String domainNamespace = null;
54+
private static String domainUid = "lboci-domain";
55+
private static ConditionFactory withStandardRetryPolicy = null;
56+
57+
// constants for creating domain image using model in image
58+
private static final String SAMPLE_APP_NAME = "sample-app";
59+
private static String clusterName = "cluster-1";
60+
private static LoggingFacade logger = null;
61+
private static String loadBalancerIP = null;
62+
private static final String OCI_LB_NAME = "ocilb";
63+
64+
/**
65+
* Install and verify operator.
66+
*
67+
* @param namespaces list of namespaces created by the IntegrationTestWatcher by the
68+
* JUnit engine parameter resolution mechanism
69+
*/
70+
@BeforeAll
71+
public static void initAll(@Namespaces(2) List<String> namespaces) {
72+
73+
logger = getLogger();
74+
// create standard, reusable retry/backoff policy
75+
withStandardRetryPolicy = with().pollDelay(2, SECONDS)
76+
.and().with().pollInterval(10, SECONDS)
77+
.atMost(5, MINUTES).await();
78+
79+
logger.info("Get a unique namespace for operator");
80+
assertNotNull(namespaces.get(0), "Namespace list is null");
81+
final String opNamespace = namespaces.get(0);
82+
83+
logger.info("Get a unique namespace for WebLogic domain1");
84+
assertNotNull(namespaces.get(1), "Namespace list is null");
85+
domainNamespace = namespaces.get(1);
86+
87+
logger.info("install and verify operator");
88+
installAndVerifyOperator(opNamespace, domainNamespace);
89+
}
90+
91+
@AfterAll
92+
public void tearDownAll() {
93+
if (System.getenv("SKIP_CLEANUP") == null
94+
|| (System.getenv("SKIP_CLEANUP") != null
95+
&& System.getenv("SKIP_CLEANUP").equalsIgnoreCase("false"))) {
96+
Kubernetes.deleteService(OCI_LB_NAME, domainNamespace);
97+
}
98+
}
99+
100+
/**
101+
* Test covers basic functionality for OCI LoadBalancer .
102+
* Create domain and OCI LoadBalancer.
103+
* Check that application is accessabale via OCI LoadBalancer
104+
*/
105+
@Test
106+
@DisplayName("Test the sample-app app can be accessed"
107+
+ " from all managed servers in the domain through OCI Load Balancer.")
108+
public void testOCILoadBalancer() throws Exception {
109+
110+
// create and verify one cluster mii domain
111+
logger.info("Create domain and verify that it's running");
112+
113+
// create docker registry secret to pull the image from registry
114+
// this secret is used only for non-kind cluster
115+
logger.info("Create docker registry secret in namespace {0}", domainNamespace);
116+
createOcirRepoSecret(domainNamespace);
117+
String adminServerPodName = domainUid + "-" + ADMIN_SERVER_NAME_BASE;
118+
String managedServerPrefix = domainUid + "-" + MANAGED_SERVER_NAME_BASE;
119+
createMiiDomainAndVerify(domainNamespace,domainUid,
120+
MII_BASIC_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG, adminServerPodName,
121+
managedServerPrefix,replicaCount);
122+
123+
int clusterHttpPort = 8001;
124+
125+
assertDoesNotThrow(() -> installAndVerifyOCILoadBalancer(domainNamespace,
126+
clusterHttpPort, clusterName, domainUid, OCI_LB_NAME),
127+
"Installation of OCI Load Balancer failed");
128+
loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME);
129+
assertNotNull(loadBalancerIP, "External IP for Load Balancer is undefined");
130+
logger.info("LoadBalancer IP is " + loadBalancerIP);
131+
verifyWebAppAccessThroughOCILoadBalancer(loadBalancerIP, 2, clusterHttpPort);
132+
}
133+
134+
/**
135+
* Retreive external IP from OCI LoadBalancer.
136+
*/
137+
private static String getLoadBalancerIP(String namespace, String lbName) throws Exception {
138+
Map<String, String> labels = new HashMap<>();
139+
labels.put("loadbalancer", lbName);
140+
V1Service service = getService(lbName, labels, namespace);
141+
assertNotNull(service, "Can't find service with name " + lbName);
142+
logger.info("Found service with name {0} in {1} namespace ", lbName, namespace);
143+
List<V1LoadBalancerIngress> ingress = service.getStatus().getLoadBalancer().getIngress();
144+
if (ingress != null) {
145+
logger.info("LoadBalancer Ingress " + ingress.toString());
146+
V1LoadBalancerIngress lbIng = ingress.stream().filter(c ->
147+
!c.getIp().equals("pending")
148+
).findAny().orElse(null);
149+
if (lbIng != null) {
150+
logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp());
151+
return lbIng.getIp();
152+
}
153+
}
154+
return null;
155+
}
156+
157+
/**
158+
* Verify the sample-app app can be accessed from all managed servers in the domain through OCI Load Balancer.
159+
*/
160+
private void verifyWebAppAccessThroughOCILoadBalancer(String lbIp, int replicaCount, int httpport) {
161+
162+
List<String> managedServerNames = new ArrayList<>();
163+
for (int i = 1; i <= replicaCount; i++) {
164+
managedServerNames.add(MANAGED_SERVER_NAME_BASE + i);
165+
}
166+
167+
// check that NGINX can access the sample apps from all managed servers in the domain
168+
String curlCmd =
169+
String.format("curl --silent --show-error --noproxy '*' http://%s:%s/sample-war/index.jsp",
170+
lbIp,
171+
httpport);
172+
assertThat(callWebAppAndCheckForServerNameInResponse(curlCmd, managedServerNames, 50))
173+
.as("Verify OCI LB can access the sample-war app "
174+
+ "from all managed servers in the domain via http")
175+
.withFailMessage("OCI LB can not access the the sample-war app "
176+
+ "from one or more of the managed servers via http")
177+
.isTrue();
178+
}
179+
}

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,24 @@ public static Callable<Boolean> serviceExists(
336336
return Service.serviceExists(serviceName, label, namespace);
337337
}
338338

339+
340+
/**
341+
* Check if a service Load Balancer External IP is generated.
342+
*
343+
* @param serviceName the name of the service to check for
344+
* @param label a Map of key value pairs the service is decorated with
345+
* @param namespace in which the service is running
346+
* @return true if the service Load Balancer External IP is generated otherwise false
347+
*/
348+
public static Callable<Boolean> isOCILoadBalancerReady(
349+
String serviceName,
350+
Map<String, String> label,
351+
String namespace) {
352+
return () -> {
353+
return Kubernetes.isOCILoadBalancerReady(serviceName, label, namespace);
354+
};
355+
}
356+
339357
/**
340358
* Check a service does not exist in the specified namespace.
341359
*

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.kubernetes.client.openapi.models.V1Job;
2626
import io.kubernetes.client.openapi.models.V1JobCondition;
2727
import io.kubernetes.client.openapi.models.V1JobList;
28+
import io.kubernetes.client.openapi.models.V1LoadBalancerIngress;
2829
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList;
2930
import io.kubernetes.client.openapi.models.V1PersistentVolumeList;
3031
import io.kubernetes.client.openapi.models.V1Pod;
@@ -37,6 +38,7 @@
3738

3839
import static io.kubernetes.client.util.Yaml.dump;
3940
import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME;
41+
import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER;
4042
import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_RELEASE_NAME;
4143
import static oracle.weblogic.kubernetes.actions.TestActions.getPodRestartVersion;
4244
import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.getPodCreationTimestamp;
@@ -514,6 +516,48 @@ public static boolean doesServiceExist(
514516
return exist;
515517
}
516518

519+
520+
/**
521+
* Checks if a Kubernetes service object exists in a given namespace.
522+
* @param serviceName name of the service to check for
523+
* @param labels the key value pair with which the service is decorated with
524+
* @param namespace the namespace in which to check for the service
525+
* @return true if the service External IP is found otherwise false
526+
* @throws ApiException when there is error in querying the cluster
527+
*/
528+
public static boolean isOCILoadBalancerReady(
529+
String serviceName, Map<String, String> labels, String namespace)
530+
throws ApiException {
531+
if (!OKE_CLUSTER) {
532+
throw new ApiException("Can't create OCI Load Balancer in non OKE enviroment");
533+
}
534+
LoggingFacade logger = getLogger();
535+
V1Service service = getService(serviceName, labels, namespace);
536+
if (service != null) {
537+
logger.info("Found service with name {0} in {1} namespace ", serviceName, namespace);
538+
539+
if (service.getStatus().getLoadBalancer() != null) {
540+
logger.info("LoadBalancer Status " + service.getStatus().getLoadBalancer().toString());
541+
List<V1LoadBalancerIngress> ingress = service.getStatus().getLoadBalancer().getIngress();
542+
if (ingress != null) {
543+
logger.info("LoadBalancer Ingress " + ingress.toString());
544+
V1LoadBalancerIngress lbIng = ingress.stream().filter(c ->
545+
!c.getIp().equals("pending")
546+
).findAny().orElse(null);
547+
if (lbIng != null) {
548+
logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp());
549+
return true;
550+
}
551+
} else {
552+
logger.info("LoadBalancer does not have assigned External IP");
553+
}
554+
}
555+
} else {
556+
logger.info("Can't find service with name " + serviceName + " in namespace " + namespace);
557+
}
558+
return false;
559+
}
560+
517561
/**
518562
* Get V1Service object for the given service name, label and namespace.
519563
* @param serviceName name of the service to look for

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.concurrent.Callable;
2525

2626
import com.google.gson.JsonObject;
27+
import io.kubernetes.client.custom.IntOrString;
2728
import io.kubernetes.client.custom.Quantity;
2829
import io.kubernetes.client.custom.V1Patch;
2930
import io.kubernetes.client.openapi.ApiException;
@@ -56,8 +57,11 @@
5657
import io.kubernetes.client.openapi.models.V1Secret;
5758
import io.kubernetes.client.openapi.models.V1SecretList;
5859
import io.kubernetes.client.openapi.models.V1SecurityContext;
60+
import io.kubernetes.client.openapi.models.V1Service;
5961
import io.kubernetes.client.openapi.models.V1ServiceAccount;
6062
import io.kubernetes.client.openapi.models.V1ServiceAccountList;
63+
import io.kubernetes.client.openapi.models.V1ServicePort;
64+
import io.kubernetes.client.openapi.models.V1ServiceSpec;
6165
import io.kubernetes.client.openapi.models.V1Volume;
6266
import io.kubernetes.client.openapi.models.V1VolumeMount;
6367
import io.kubernetes.client.openapi.models.V1WeightedPodAffinityTerm;
@@ -168,6 +172,7 @@
168172
import static oracle.weblogic.kubernetes.actions.TestActions.createPersistentVolume;
169173
import static oracle.weblogic.kubernetes.actions.TestActions.createPersistentVolumeClaim;
170174
import static oracle.weblogic.kubernetes.actions.TestActions.createSecret;
175+
import static oracle.weblogic.kubernetes.actions.TestActions.createService;
171176
import static oracle.weblogic.kubernetes.actions.TestActions.createServiceAccount;
172177
import static oracle.weblogic.kubernetes.actions.TestActions.defaultAppParams;
173178
import static oracle.weblogic.kubernetes.actions.TestActions.dockerLogin;
@@ -210,6 +215,7 @@
210215
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isGrafanaReady;
211216
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed;
212217
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isNginxReady;
218+
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isOCILoadBalancerReady;
213219
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPodRestarted;
214220
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPrometheusReady;
215221
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isTraefikReady;
@@ -670,6 +676,65 @@ public static boolean upgradeAndVerifyOperator(String opNamespace, OperatorParam
670676
return true;
671677
}
672678

679+
680+
/**
681+
* Install OCI Load Balancer and wait up to five minutes until the External IP is ready.
682+
*
683+
* @param namespace the namespace in which the Noci Load Balancer will be installed
684+
* @param portshttp the http port of oci load balancer
685+
* @param clusterName name of WLS cluster
686+
* @param domainUID domain UID
687+
* @param loadBalancerName service name for OCI Load Balancer
688+
*/
689+
public static void installAndVerifyOCILoadBalancer(
690+
String namespace,
691+
int portshttp,
692+
String clusterName,
693+
String domainUID,
694+
String loadBalancerName) throws ApiException {
695+
Map<String, String> annotations = new HashMap<>();
696+
annotations.put("service.beta.kubernetes.io/oci-load-balancer-shape", "400Mbps");
697+
Map<String, String> selectors = new HashMap<>();
698+
Map<String, String> labels = new HashMap<>();
699+
labels.put("loadbalancer", loadBalancerName);
700+
selectors.put("weblogic.clusterName", clusterName);
701+
selectors.put("weblogic.domainUID",domainUID);
702+
List<V1ServicePort> ports = new ArrayList<>();
703+
ports.add(new V1ServicePort()
704+
.name("http")
705+
.port(portshttp)
706+
.targetPort(new IntOrString(portshttp))
707+
.protocol("TCP"));
708+
709+
V1Service service = new V1Service()
710+
.metadata(new V1ObjectMeta()
711+
.name(loadBalancerName)
712+
.namespace(namespace)
713+
.labels(labels)
714+
.annotations(annotations))
715+
.spec(new V1ServiceSpec()
716+
.ports(ports)
717+
.selector(selectors)
718+
.sessionAffinity("None")
719+
.type("LoadBalancer"));
720+
LoggingFacade logger = getLogger();
721+
assertNotNull(service, "Can't create ocilb service, returns null");
722+
assertDoesNotThrow(() -> createService(service), "Can't create OCI LoadBalancer service");
723+
checkServiceExists(loadBalancerName,namespace);
724+
725+
// wait until the external IP is generated.
726+
withStandardRetryPolicy
727+
.conditionEvaluationListener(
728+
condition -> logger.info(
729+
"Waiting for external IP to be generated in {0} (elapsed time {1}ms, remaining time {2}ms)",
730+
namespace,
731+
condition.getElapsedTimeInMS(),
732+
condition.getRemainingTimeInMS()))
733+
.until(assertDoesNotThrow(() -> isOCILoadBalancerReady(
734+
loadBalancerName,
735+
labels, namespace), "isOCILoadBalancerReady failed with ApiException"));
736+
}
737+
673738
/**
674739
* Install NGINX and wait up to five minutes until the NGINX pod is ready.
675740
*
@@ -3790,5 +3855,4 @@ public static Callable<Boolean> runClientInsidePod(String podName, String namesp
37903855
return ((result.exitValue() == 0));
37913856
});
37923857
}
3793-
37943858
}

‎integration-tests/src/test/resources/wdt-models/model.sessmigr.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ topology:
2222
ListenPort: 7001
2323
ServerTemplate:
2424
"cluster-1-template":
25+
NetworkAccessPoint:
26+
T3Channel:
27+
PublicPort: 30012
28+
ListenPort: 30012
29+
PublicAddress: kubernetes
2530
Cluster: "cluster-1"
2631

2732
appDeployments:

0 commit comments

Comments
 (0)