Skip to content

Commit c9a911d

Browse files
kavyashree-rdivyenpatel
kavyashree-r
authored and
divyenpatel
committed
WCP-pre-data (kubernetes-sigs#3772)
wcp-pre-data
1 parent 371d297 commit c9a911d

File tree

1 file changed

+333
-0
lines changed

1 file changed

+333
-0
lines changed

‎tests/e2e/predataForWCP.go‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"os"
22+
23+
snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
24+
snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned"
25+
"github.com/onsi/ginkgo/v2"
26+
"github.com/onsi/gomega"
27+
vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
28+
vmopv2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
29+
vmopv3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3"
30+
vmopv4 "github.com/vmware-tanzu/vm-operator/api/v1alpha4"
31+
v1 "k8s.io/api/core/v1"
32+
apierrors "k8s.io/apimachinery/pkg/api/errors"
33+
"k8s.io/apimachinery/pkg/api/resource"
34+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35+
"k8s.io/apimachinery/pkg/runtime"
36+
clientset "k8s.io/client-go/kubernetes"
37+
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
38+
"k8s.io/kubernetes/test/e2e/framework"
39+
fpod "k8s.io/kubernetes/test/e2e/framework/pod"
40+
fpv "k8s.io/kubernetes/test/e2e/framework/pv"
41+
fss "k8s.io/kubernetes/test/e2e/framework/statefulset"
42+
admissionapi "k8s.io/pod-security-admission/api"
43+
ctlrclient "sigs.k8s.io/controller-runtime/pkg/client"
44+
45+
cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator"
46+
k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes"
47+
)
48+
49+
var _ = ginkgo.Describe("WCP-predata",
50+
ginkgo.Label(p0, wcp, core), func() {
51+
52+
f := framework.NewDefaultFramework("supervisor-predata")
53+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
54+
55+
var (
56+
client clientset.Interface
57+
namespace string
58+
storagePolicyName string
59+
scParameters map[string]string
60+
vmClass string
61+
statuscode int
62+
)
63+
64+
ginkgo.BeforeEach(func() {
65+
ctx, cancel := context.WithCancel(context.Background())
66+
defer cancel()
67+
68+
client = f.ClientSet
69+
bootstrap()
70+
71+
// Skip if not supervisor cluster
72+
if !supervisorCluster {
73+
ginkgo.Skip("Test is only for Supervisor Cluster")
74+
}
75+
76+
scParameters = make(map[string]string)
77+
storagePolicyName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores)
78+
profileID := e2eVSphere.GetSpbmPolicyID(storagePolicyName)
79+
scParameters[scParamStoragePolicyID] = profileID
80+
81+
//datastoreURL is required to get dsRef ID which is used to get contentLibId
82+
datastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL)
83+
dsRef := getDsMoRefFromURL(ctx, datastoreURL)
84+
framework.Logf("dsmoId: %v", dsRef.Value)
85+
86+
vcRestSessionId := createVcSession4RestApis(ctx)
87+
contentLibId, err := createAndOrGetContentlibId4Url(vcRestSessionId, GetAndExpectStringEnvVar(envContentLibraryUrl),
88+
dsRef.Value, &e2eVSphere)
89+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
90+
91+
framework.Logf("Create a WCP namespace for the test")
92+
vmClass := os.Getenv(envVMClass)
93+
if vmClass == "" {
94+
vmClass = vmClassBestEffortSmall
95+
}
96+
97+
// Create SVC namespace and assign storage policy and vmContent Library
98+
namespace, statuscode, err = createtWcpNsWithZonesAndPolicies(vcRestSessionId,
99+
[]string{profileID}, getSvcId(vcRestSessionId, &e2eVSphere),
100+
nil, vmClass, contentLibId)
101+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
102+
gomega.Expect(statuscode).To(gomega.Equal(status_code_success))
103+
})
104+
105+
/*
106+
Supervisor Cluster Pre-Data Setup
107+
108+
Creates resources WITHOUT cleanup:
109+
- 5 standalone PVCs with Pods
110+
- 1 StatefulSet with 5 replicas
111+
- 3 VolumeSnapshots
112+
- 2 VMService VMs
113+
114+
Total: 12 PVCs (5 standalone + 5 StatefulSet + 2 VM PVCs)
115+
*/
116+
ginkgo.It("Create 5 PVCs, 1 StatefulSet (5 replicas), 3 Snapshots, and 2 VMs without cleanup",
117+
func() {
118+
ctx, cancel := context.WithCancel(context.Background())
119+
defer cancel()
120+
121+
framework.Logf("=== Supervisor Pre-Data Setup Started ===")
122+
framework.Logf("Namespace: %s", namespace)
123+
framework.Logf("Storage Policy: %s", storagePolicyName)
124+
125+
// Get storage class
126+
storageclass, err := client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{})
127+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
128+
129+
// Initialize clients
130+
var snapc *snapclient.Clientset
131+
restConfig = getRestConfigClient()
132+
snapc, err = snapclient.NewForConfig(restConfig)
133+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
134+
135+
cnsopC, err := k8s.NewClientForGroup(ctx, restConfig, cnsoperatorv1alpha1.GroupName)
136+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
137+
138+
// ==============================================
139+
// Section 1: Create 5 Standalone PVCs with Pods
140+
// ==============================================
141+
ginkgo.By("Step 1: Creating 5 standalone PVCs with Pods")
142+
143+
standalonePVCs := make([]*v1.PersistentVolumeClaim, 5)
144+
standalonePods := make([]*v1.Pod, 5)
145+
146+
for i := 0; i < 5; i++ {
147+
pvc, err := createPVC(ctx, client, namespace, nil, diskSize, storageclass, v1.ReadWriteOnce)
148+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
149+
standalonePVCs[i] = pvc
150+
framework.Logf("Created PVC: %s", pvc.Name)
151+
}
152+
153+
// Wait for PVCs to be bound
154+
standalonePVs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, standalonePVCs,
155+
framework.ClaimProvisionTimeout)
156+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
157+
framework.Logf("All 5 standalone PVCs are bound")
158+
159+
// Verify CNS metadata
160+
for i := 0; i < 5; i++ {
161+
volHandle := standalonePVs[i].Spec.CSI.VolumeHandle
162+
err = waitAndVerifyCnsVolumeMetadata(ctx, volHandle, standalonePVCs[i], standalonePVs[i], nil)
163+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
164+
}
165+
166+
// Create pods
167+
for i := 0; i < 5; i++ {
168+
pod, err := createPod(ctx, client, namespace, nil,
169+
[]*v1.PersistentVolumeClaim{standalonePVCs[i]}, false, "")
170+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
171+
standalonePods[i] = pod
172+
framework.Logf("Created Pod: %s", pod.Name)
173+
}
174+
175+
// Wait for pods to be running
176+
for i := 0; i < 5; i++ {
177+
err = fpod.WaitForPodRunningInNamespace(ctx, client, standalonePods[i])
178+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
179+
}
180+
framework.Logf("All 5 standalone pods are running")
181+
182+
// ==============================================
183+
// Section 2: Create StatefulSet with 5 replicas
184+
// ==============================================
185+
ginkgo.By("Step 2: Creating StatefulSet with 5 replicas")
186+
187+
// Check if nginx service already exists
188+
var service *v1.Service
189+
existingService, err := client.CoreV1().Services(namespace).Get(ctx, servicename, metav1.GetOptions{})
190+
if err != nil {
191+
if apierrors.IsNotFound(err) {
192+
// Service doesn't exist, create it
193+
service = CreateService(namespace, client)
194+
framework.Logf("Service created: %s", service.Name)
195+
} else {
196+
// Some other error occurred
197+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
198+
}
199+
} else {
200+
// Service already exists
201+
framework.Logf("Service '%s' already exists in namespace '%s', reusing it", existingService.Name, namespace)
202+
service = existingService
203+
}
204+
205+
statefulset := GetStatefulSetFromManifest(namespace)
206+
replicas := int32(3)
207+
statefulset.Spec.Replicas = &replicas
208+
statefulset.Spec.VolumeClaimTemplates[0].Spec.StorageClassName = &storagePolicyName
209+
statefulset.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[v1.ResourceStorage] =
210+
resource.MustParse(diskSize)
211+
212+
CreateStatefulSet(namespace, statefulset, client)
213+
framework.Logf("StatefulSet created: %s with 3 replicas", statefulset.Name)
214+
215+
// Wait for StatefulSet to be ready
216+
fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas)
217+
gomega.Expect(fss.CheckMount(ctx, client, statefulset, mountPath)).NotTo(gomega.HaveOccurred())
218+
219+
ssPodsAfterCreation, err := fss.GetPodList(ctx, client, statefulset)
220+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
221+
gomega.Expect(len(ssPodsAfterCreation.Items)).To(gomega.Equal(int(replicas)))
222+
framework.Logf("StatefulSet is ready with 5 replicas")
223+
224+
// ==============================================
225+
// Section 3: Create 3 VolumeSnapshots
226+
// ==============================================
227+
228+
vcVersion = getVCversion(ctx, vcAddress)
229+
isVC90 := isVersionGreaterOrEqual(vcVersion, quotaSupportedVCVersion)
230+
231+
if isVC90 {
232+
ginkgo.By("Step 3: Creating 3 VolumeSnapshots")
233+
234+
volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy)
235+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
236+
framework.Logf("VolumeSnapshotClass created: %s", volumeSnapshotClass.Name)
237+
238+
volumeSnapshots := make([]*snapV1.VolumeSnapshot, 3)
239+
for i := 0; i < 3; i++ {
240+
snapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx,
241+
getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, standalonePVCs[i].Name),
242+
metav1.CreateOptions{})
243+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
244+
volumeSnapshots[i] = snapshot
245+
framework.Logf("Created VolumeSnapshot: %s from PVC: %s",
246+
snapshot.Name, standalonePVCs[i].Name)
247+
}
248+
249+
// Wait for snapshots to be ready
250+
for i := 0; i < 3; i++ {
251+
snapshot, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshots[i].Name)
252+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
253+
gomega.Expect(snapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero())
254+
framework.Logf("VolumeSnapshot %s is ready", volumeSnapshots[i].Name)
255+
}
256+
}
257+
258+
// ==============================================
259+
// Section 4: Create 2 VMService VMs
260+
// ==============================================
261+
ginkgo.By("Step 4: Creating 2 VMService VMs")
262+
263+
vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
264+
vmopScheme := runtime.NewScheme()
265+
gomega.Expect(vmopv1.AddToScheme(vmopScheme)).Should(gomega.Succeed())
266+
gomega.Expect(vmopv2.AddToScheme(vmopScheme)).Should(gomega.Succeed())
267+
gomega.Expect(vmopv3.AddToScheme(vmopScheme)).Should(gomega.Succeed())
268+
gomega.Expect(vmopv4.AddToScheme(vmopScheme)).Should(gomega.Succeed())
269+
vmopC, err := ctlrclient.New(f.ClientConfig(), ctlrclient.Options{Scheme: vmopScheme})
270+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
271+
vmi := waitNGetVmiForImageName(ctx, vmopC, vmImageName)
272+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
273+
framework.Logf("Found VirtualMachineImage: %s", vmi)
274+
275+
secretName := createBootstrapSecretForVmsvcVms(ctx, client, namespace)
276+
framework.Logf("Created bootstrap secret: %s", secretName)
277+
278+
vmPVCs := make([]*v1.PersistentVolumeClaim, 2)
279+
vms := make([]*vmopv1.VirtualMachine, 2)
280+
281+
for i := 0; i < 2; i++ {
282+
// Create PVC for VM
283+
vmPvc, err := createPVC(ctx, client, namespace, nil, "2Gi", storageclass, v1.ReadWriteOnce)
284+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
285+
vmPVCs[i] = vmPvc
286+
framework.Logf("Created VM PVC: %s", vmPvc.Name)
287+
288+
// Wait for PVC to be bound
289+
_, err = fpv.WaitForPVClaimBoundPhase(ctx, client,
290+
[]*v1.PersistentVolumeClaim{vmPVCs[i]}, framework.ClaimProvisionTimeout)
291+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
292+
293+
// Create VM
294+
vm := createVmServiceVmWithPvcs(ctx, vmopC, namespace, vmClass,
295+
[]*v1.PersistentVolumeClaim{vmPVCs[i]}, vmi, storagePolicyName, secretName)
296+
vms[i] = vm
297+
framework.Logf("Created VMService VM: %s", vm.Name)
298+
299+
// Verify PVC attached to VM
300+
err = waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vms[i],
301+
[]*v1.PersistentVolumeClaim{vmPVCs[i]})
302+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
303+
framework.Logf("Verified PVC attached to VM: %s", vms[i].Name)
304+
}
305+
306+
// ==============================================
307+
// Final Summary
308+
// ==============================================
309+
ginkgo.By("Step 5: Verifying all resources")
310+
311+
framework.Logf("=== Supervisor Pre-Data Setup Complete ===")
312+
framework.Logf("Namespace: %s", namespace)
313+
framework.Logf("Storage Policy: %s", storagePolicyName)
314+
framework.Logf("")
315+
framework.Logf("Resources Created:")
316+
framework.Logf(" - Standalone PVCs: 3 (3 x 2Gi = 6Gi)")
317+
framework.Logf(" - Standalone Pods: 3")
318+
framework.Logf(" - StatefulSet: 1 with 3 replicas (3 x 2Gi = 6Gi)")
319+
framework.Logf(" - VolumeSnapshots: 3 (~6Gi)")
320+
framework.Logf(" - VMService VMs: 2 (2 x 10Gi = 20Gi)")
321+
framework.Logf(" - VM PVCs: 2")
322+
framework.Logf("")
323+
framework.Logf("Total Resources:")
324+
framework.Logf(" - PVCs: 12 (5 standalone + 5 StatefulSet + 2 VM)")
325+
framework.Logf(" - Pods: 10 (5 standalone + 5 StatefulSet)")
326+
framework.Logf(" - VMs: 2")
327+
framework.Logf(" - Snapshots: 3")
328+
framework.Logf(" - Total Storage: ~46Gi")
329+
framework.Logf("==========================================")
330+
331+
ginkgo.By("*** RESOURCES NOT CLEANED UP - Will persist after test ***")
332+
})
333+
})

0 commit comments

Comments
 (0)