Skip to content

Commit f158e0d

Browse files
z103cbz103cb
authored
Create e2e test to emulate deployment use cases for MCAD (#400)
* Create e2e test to emulate deployment use cases for MCAD Changes to e2e script Setting up scenario one * Added cleanup step * Added more files * Added additional steps Removed files * Checkpoint commit. * Second scenario draft. * Added scenario 3 * Updates to deployment-03 steps * Modified scenario 1 to use single quota tree. Exposes the behaviour in #409 * Update scenario 2 and 3 with default quotas * Improvements to end to end tests script Small changes to existing e2e tests. * Small updates * Updates to scenario2 * Updates to scenario01 * Resolve MCAD is in crash loop backoff mode after deploying it in quota management mode #409 * White space cleanup. Test changes. * Fixed small issues. * Validated that that default quota works when multiple trees are available
1 parent 089cf9f commit f158e0d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+1442
-51
lines changed

‎hack/run-e2e-kind.sh‎

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,8 @@ export MCAD_IMAGE_PULL_POLICY="${3-Always}"
4444
export IMAGE_MCAD="${IMAGE_REPOSITORY_MCAD}:${IMAGE_TAG_MCAD}"
4545
CLUSTER_STARTED="false"
4646
export KUTTL_VERSION=0.15.0
47-
export KUTTL_TEST_OPT="--config ${ROOT_DIR}/kuttl-test.yaml"
48-
# FOR DEBUGGING
49-
#export KUTTL_TEST_OPT="--config ${ROOT_DIR}/kuttl-test.yaml --skip-delete"
47+
export KUTTL_OPTIONS=${TEST_KUTTL_OPTIONS}
48+
export KUTTL_TEST_SUITES=("${ROOT_DIR}/test/kuttl-test.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-03.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-02.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-01.yaml")
5049

5150
function update_test_host {
5251

@@ -310,15 +309,14 @@ function cleanup {
310309
fi
311310
}
312311

313-
function mcad-quota-management-down {
314-
312+
function undeploy_mcad_helm {
315313
# Helm chart install name
316314
local helm_chart_name=$(helm list -n kube-system --short | grep mcad-controller)
317315

318316
# start mcad controller
319317
echo "Stopping MCAD Controller for Quota Management Testing..."
320318
echo "helm delete ${helm_chart_name}"
321-
helm delete -n kube-system ${helm_chart_name}
319+
helm delete -n kube-system ${helm_chart_name} --wait
322320
if [ $? -ne 0 ]
323321
then
324322
echo "Failed to undeploy controller"
@@ -372,16 +370,24 @@ function setup-mcad-env {
372370
}
373371

374372
function kuttl-tests {
375-
echo "kubectl kuttl test ${KUTTL_TEST_OPT}"
376-
kubectl kuttl test ${KUTTL_TEST_OPT}
377-
if [ $? -ne 0 ]
378-
then
379-
echo "quota management kuttl e2e tests failure, exiting."
380-
exit 1
381-
else
382-
# Takes a bit of time for namespace created in kuttl testing to completely delete.
383-
sleep 40
384-
fi
373+
for kuttl_test in ${KUTTL_TEST_SUITES[@]}; do
374+
echo "kubectl kuttl test --config ${kuttl_test}"
375+
kubectl kuttl test --config ${kuttl_test}
376+
if [ $? -ne 0 ]
377+
then
378+
echo "kuttl e2e test '${kuttl_test}' failure, exiting."
379+
exit 1
380+
fi
381+
#clean up after sucessfull execution of a test by removing all quota subtrees
382+
#and undeploying mcad helm chart.
383+
kubectl delete quotasubtrees -n kube-system --all --wait
384+
if [ $? -ne 0 ]
385+
then
386+
echo "Failed to delete quotasubtrees for test: '${kuttl_test}'"
387+
exit 1
388+
fi
389+
undeploy_mcad_helm
390+
done
385391
rm -f kubeconfig
386392
}
387393

@@ -392,6 +398,5 @@ kind-up-cluster
392398
setup-mcad-env
393399
# MCAD with quotamanagement options is started by kuttl-tests
394400
kuttl-tests
395-
mcad-quota-management-down
396401
mcad-up
397402
go test ./test/e2e -v -timeout 130m -count=1

‎pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func NewQuotaManager(dispatchedAWDemands map[string]*clusterstateapi.Resource, d
128128

129129
// Set the name of the forest in the backend
130130
qm.quotaManagerBackend.AddForest(QuotaManagerForestName)
131-
klog.V(10).Infof("[NewQuotaManager] Before initialization QuotaSubtree informer - %s", qm.quotaManagerBackend.String())
131+
klog.V(4).Infof("[NewQuotaManager] Before initialization QuotaSubtree informer - %s", qm.quotaManagerBackend.String())
132132

133133
// Create a resource plan manager
134134
qm.quotaSubtreeManager, err = qstmanager.NewQuotaSubtreeManager(config, qm.quotaManagerBackend)
@@ -171,19 +171,47 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
171171
klog.V(4).Infof("[loadDispatchedAWs] No dispatched AppWrappers found to preload.")
172172
return nil
173173
}
174+
allTrees := qm.GetValidQuotaLabels()
175+
klog.V(4).Infof("[loadDispatchedAWs] valid quota labels:%v", allTrees)
176+
if len(allTrees) == 0 && len(dispatchedAWs) > 0 {
177+
klog.Warning("[loadDispatchedAWs] No quota trees are defined in the cluster.")
178+
klog.Warning("[loadDispatchedAWs] The resources for the following app wrappers will not be counted in the quota tree:")
179+
for k := range dispatchedAWDemands {
180+
aw := getDispatchedAppWrapper(dispatchedAWs, k)
181+
if aw != nil {
182+
klog.Warningf("[loadDispatchedAWs] app wrapper %s/%s not counted. AW labels: %v", aw.Namespace, aw.Name, aw.GetLabels())
183+
}
184+
}
185+
return nil
186+
}
174187

175188
// Process list of AppWrappers that are already dispatched
176189
var result *multierror.Error
177190

178191
for k, v := range dispatchedAWDemands {
179192
aw := getDispatchedAppWrapper(dispatchedAWs, k)
180193
if aw != nil {
194+
klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s found to preload.", aw.Namespace, aw.Name)
195+
newLabels := make(map[string]string)
196+
for key, value := range aw.Labels {
197+
newLabels[key] = value
198+
}
199+
for _, treeName := range allTrees {
200+
if _, quotaSetForAW := newLabels[treeName]; !quotaSetForAW {
201+
newLabels[treeName] = "default"
202+
klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s adding default quota labels.", aw.Namespace, aw.Name)
203+
}
204+
205+
}
206+
aw.SetLabels(newLabels)
207+
181208
doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil)
182209
if !doesFit {
183210
klog.Errorf("[loadDispatchedAWs] Loading of AppWrapper %s/%s failed.",
184211
aw.Namespace, aw.Name)
185212
result = multierror.Append(result, fmt.Errorf("loading of AppWrapper %s/%s failed, %s",
186213
aw.Namespace, aw.Name, errorMessage))
214+
qm.Release(aw)
187215
}
188216

189217
if len(preemptionIds) > 0 {
@@ -192,7 +220,6 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
192220
result = multierror.Append(result, fmt.Errorf("loading of AppWrapper %s/%s caused invalid preemptions: %v. Quota Manager is in inconsistent state",
193221
aw.Namespace, aw.Name, preemptionIds))
194222
}
195-
klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s found to preload.", aw.Namespace, aw.Name)
196223
} else {
197224
klog.Warningf("[loadDispatchedAWs] Unable to obtain AppWrapper from key: %s. Loading of AppWrapper will be skipped.", k)
198225
}

‎pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr/quotasubtmgr/event_handlers.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func (qstm *QuotaSubtreeManager) addQST(obj interface{}) {
3333
qstm.qstMap[qst.Namespace+"/"+qst.Name] = qst
3434
qstm.setQuotasubtreeChanged()
3535
qstm.qstMutex.Unlock()
36-
klog.V(10).Infof("[addQST] Add complete for: %s/%s", qst.Name, qst.Namespace)
36+
klog.V(4).Infof("[addQST] Add complete for: %s/%s", qst.Name, qst.Namespace)
3737
}
3838

3939
func (qstm *QuotaSubtreeManager) updateQST(oldObj, newObj interface{}) {
@@ -65,7 +65,7 @@ func (qstm *QuotaSubtreeManager) updateQST(oldObj, newObj interface{}) {
6565
qstm.setQuotasubtreeChanged()
6666
qstm.qstMutex.Unlock()
6767
}
68-
klog.V(10).Infof("[updateQST] Update complete for: %s/%s", newQST.Name, newQST.Namespace)
68+
klog.V(4).Infof("[updateQST] Update complete for: %s/%s", newQST.Name, newQST.Namespace)
6969
}
7070

7171
func (qstm *QuotaSubtreeManager) deleteQST(obj interface{}) {
@@ -79,5 +79,5 @@ func (qstm *QuotaSubtreeManager) deleteQST(obj interface{}) {
7979

8080
delete(qstm.qstMap, string(qst.UID))
8181
delete(qstm.qstMap, qst.Namespace+"/"+qst.Name)
82-
klog.V(10).Infof("[deleteQST] Delete complete for: %s/%s", qst.Name, qst.Namespace)
82+
klog.V(4).Infof("[deleteQST] Delete complete for: %s/%s", qst.Name, qst.Namespace)
8383
}

‎pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr/quotasubtmgr/quota_subtree_manager.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ func newQuotaSubtreeManager(config *rest.Config, quotaManagerBackend *qmlib.Mana
6161
qstm := &QuotaSubtreeManager{
6262
quotaManagerBackend: quotaManagerBackend,
6363
qstMap: make(map[string]*qstv1.QuotaSubtree),
64+
qstChanged: true,
6465
}
6566
// QuotaSubtree informer setup
6667
qstClient, err := qst.NewForConfigOrDie(config)
@@ -83,19 +84,19 @@ func newQuotaSubtreeManager(config *rest.Config, quotaManagerBackend *qmlib.Mana
8384

8485
// Start resource plan informers
8586
neverStop := make(chan struct{})
86-
klog.V(10).Infof("[newQuotaSubtreeManager] Starting QuotaSubtree Informer.")
87+
klog.V(4).Infof("[newQuotaSubtreeManager] Starting QuotaSubtree Informer.")
8788
go qstm.quotaSubtreeInformer.Informer().Run(neverStop)
8889

8990
// Wait for cache sync
90-
klog.V(10).Infof("[newQuotaSubtreeManager] Waiting for QuotaSubtree informer cache sync. to complete.")
91+
klog.V(4).Infof("[newQuotaSubtreeManager] Waiting for QuotaSubtree informer cache sync. to complete.")
9192
qstm.qstSynced = qstm.quotaSubtreeInformer.Informer().HasSynced
9293
if !cache.WaitForCacheSync(neverStop, qstm.qstSynced) {
9394
return nil, errors.New("failed to wait for the quota sub tree informer to synch")
9495
}
9596

9697
// Initialize Quota Trees
9798
qstm.initializeQuotaTreeBackend()
98-
klog.V(10).Infof("[newQuotaSubtreeManager] QuotaSubtree Manager initialization complete.")
99+
klog.V(4).Infof("[newQuotaSubtreeManager] QuotaSubtree Manager initialization complete.")
99100
return qstm, nil
100101
}
101102

@@ -129,7 +130,7 @@ func (qstm *QuotaSubtreeManager) clearQuotasubtreeChanged() {
129130
func (qstm *QuotaSubtreeManager) IsQuotasubtreeChanged() bool {
130131
qstm.qstMutex.RLock()
131132
defer qstm.qstMutex.RUnlock()
132-
133+
klog.V(4).Infof("[IsQuotasubtreeChanged] QuotaSubtree Manager changed %t.", qstm.qstChanged)
133134
return qstm.qstChanged
134135
}
135136

@@ -178,7 +179,7 @@ func (qstm *QuotaSubtreeManager) createTreeNodesFromQST(qst *qstv1.QuotaSubtree)
178179
Quota: quota,
179180
Hard: strconv.FormatBool(qstChild.Quotas.HardLimit),
180181
}
181-
klog.V(10).Infof("[createTreeNodesFromQST] Created node: %s=%#v for QuotaSubtree %s completed.",
182+
klog.V(4).Infof("[createTreeNodesFromQST] Created node: %s=%#v for QuotaSubtree %s completed.",
182183
child_key, *node, qst.Name)
183184

184185
//Add to the list of nodes from this quotasubtree

‎test/e2e-kuttl-deployment-01/steps/00-assert.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Verify CRDs existence
2+
apiVersion: apiextensions.k8s.io/v1
3+
kind: CustomResourceDefinition
4+
metadata:
5+
name: appwrappers.mcad.ibm.com
6+
status:
7+
acceptedNames:
8+
kind: AppWrapper
9+
listKind: AppWrapperList
10+
plural: appwrappers
11+
singular: appwrapper
12+
storedVersions:
13+
- v1beta1
14+
---
15+
# Verify test namespace existence
16+
apiVersion: v1
17+
kind: Namespace
18+
metadata:
19+
name: start-up

‎test/e2e-kuttl-deployment-01/steps/00-install.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: start-up

‎test/e2e-kuttl-deployment-01/steps/01-assert.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Verify AppWrapper was dispatched and pod was created
2+
apiVersion: mcad.ibm.com/v1beta1
3+
kind: AppWrapper
4+
metadata:
5+
name: no-quota-deployment-01
6+
namespace: start-up
7+
status:
8+
state: Running
9+
---
10+
apiVersion: apps/v1
11+
kind: Deployment
12+
metadata:
13+
name: no-quota-deployment-01
14+
namespace: start-up
15+
labels:
16+
app: no-quota-deployment-01
17+
appwrapper.mcad.ibm.com: no-quota-deployment-01
18+
resourceName: no-quota-deployment-01
19+
status:
20+
availableReplicas: 1
21+
observedGeneration: 1
22+
readyReplicas: 1
23+
replicas: 1
24+
updatedReplicas: 1

‎test/e2e-kuttl-deployment-01/steps/01-install.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
apiVersion: mcad.ibm.com/v1beta1
2+
kind: AppWrapper
3+
metadata:
4+
name: no-quota-deployment-01
5+
namespace: start-up
6+
spec:
7+
resources:
8+
GenericItems:
9+
- replicas: 1
10+
generictemplate:
11+
apiVersion: apps/v1
12+
kind: Deployment
13+
metadata:
14+
name: no-quota-deployment-01
15+
namespace: start-up
16+
labels:
17+
app: no-quota-deployment-01
18+
spec:
19+
selector:
20+
matchLabels:
21+
app: no-quota-deployment-01
22+
replicas: 1
23+
template:
24+
metadata:
25+
labels:
26+
app: deployment-echoserver-01
27+
spec:
28+
containers:
29+
- name: no-quota-deployment-01
30+
image: kicbase/echo-server:1.0
31+
ports:
32+
- containerPort: 80
33+
resources:
34+
requests:
35+
cpu: 300m
36+
memory: 32Mi
37+
limits:
38+
cpu: 300m
39+
memory: 32Mi

‎test/e2e-kuttl-deployment-01/steps/02-assert.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: mcad.ibm.com/v1beta1
3+
kind: AppWrapper
4+
metadata:
5+
name: no-quota-job-02
6+
namespace: start-up
7+
status:
8+
state: Completed

‎test/e2e-kuttl-deployment-01/steps/02-install.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
apiVersion: mcad.ibm.com/v1beta1
2+
kind: AppWrapper
3+
metadata:
4+
name: no-quota-job-02
5+
namespace: start-up
6+
spec:
7+
schedulingSpec:
8+
minAvailable: 1
9+
resources:
10+
GenericItems:
11+
- replicas: 1
12+
completionstatus: Complete
13+
custompodresources:
14+
- replicas: 1
15+
requests:
16+
cpu: 500m
17+
nvidia.com/gpu: 0
18+
memory: 300Mi
19+
limits:
20+
cpu: 500m
21+
nvidia.com/gpu: 0
22+
memory: 300Mi
23+
generictemplate:
24+
apiVersion: batch/v1
25+
kind: Job
26+
metadata:
27+
name: no-quota-job-02
28+
namespace: start-up
29+
labels:
30+
appwrapper.mcad.ibm.com: no-quota-job-02
31+
spec:
32+
parallelism: 1
33+
completions: 1
34+
template:
35+
metadata:
36+
name: no-quota-job-1
37+
namespace: start-up
38+
labels:
39+
appwrapper.mcad.ibm.com: no-quota-job-02
40+
spec:
41+
terminationGracePeriodSeconds: 1
42+
restartPolicy: Never
43+
containers:
44+
- name: ubuntu
45+
image: ubuntu:latest
46+
imagePullPolicy: IfNotPresent
47+
command:
48+
- sh
49+
- -c
50+
- |
51+
sleep 30
52+
resources:
53+
requests:
54+
cpu: 500m
55+
nvidia.com/gpu: 0
56+
memory: 300Mi
57+
limits:
58+
cpu: 500m
59+
nvidia.com/gpu: 0
60+
memory: 300Mi

0 commit comments

Comments
 (0)