Skip to content

Commit 50b6fa3

Browse files
rjrudinrjrudin
committed
Splitting up functional tests
I could not get the "fragile" tests to pass when they're run with all the "fast" functional tests. I'm taking the L and running the "fragile" ones first, then the fast ones, and then the slow ones. Also got rid of some duplication in Jenkinsfile by making the 11.0 regression build use "runtests".
1 parent bb45c6b commit 50b6fa3

File tree

2 files changed

+73
-61
lines changed

2 files changed

+73
-61
lines changed

‎Jenkinsfile‎

Lines changed: 50 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,21 @@ def runtests(String type, String version){
55
setUpML '$WORKSPACE/xdmp/src/Mark*.rpm'
66
copyConvertersRPM type,version
77
setUpMLConverters '$WORKSPACE/xdmp/src/Mark*Converters*.rpm'
8-
sh label:'deploy project', script: '''#!/bin/bash
8+
sh label:'deploy test app', script: '''#!/bin/bash
99
export JAVA_HOME=$JAVA_HOME_DIR
1010
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
1111
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
1212
cd java-client-api
13-
./gradlew mlDeploy
13+
./gradlew -i mlDeploy -PmlForestDataDirectory=/space
1414
'''
15-
sh label:'marklogic client test', script: '''#!/bin/bash
15+
sh label:'run marklogic-client-api tests', script: '''#!/bin/bash
1616
export JAVA_HOME=$JAVA_HOME_DIR
1717
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
1818
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
1919
cd java-client-api
2020
./gradlew marklogic-client-api:test || true
2121
'''
22-
sh label:'ml development tool test', script: '''#!/bin/bash
22+
sh label:'run ml-development-tools tests', script: '''#!/bin/bash
2323
export JAVA_HOME=$JAVA_HOME_DIR
2424
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
2525
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
@@ -28,14 +28,12 @@ def runtests(String type, String version){
2828
./gradlew ml-development-tools:generateTests || true
2929
./gradlew ml-development-tools:test || true
3030
'''
31-
sh label:'test', script: '''#!/bin/bash
31+
sh label:'run functional tests', script: '''#!/bin/bash
3232
export JAVA_HOME=$JAVA_HOME_DIR
3333
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
3434
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
3535
cd java-client-api
36-
./gradlew mlDeploy
37-
./gradlew marklogic-client-api-functionaltests:testFastFunctionalTests || true
38-
./gradlew marklogic-client-api-functionaltests:testSlowFunctionalTests || true
36+
./gradlew marklogic-client-api-functionaltests:runFunctionalTests || true
3937
'''
4038
sh label:'post-test-process', script: '''
4139
cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/
@@ -51,7 +49,7 @@ pipeline{
5149
agent {label 'javaClientLinuxPool'}
5250
options {
5351
checkoutToSubdirectory 'java-client-api'
54-
buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '30', numToKeepStr: '')
52+
buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10')
5553
}
5654
parameters{
5755
booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions')
@@ -63,41 +61,31 @@ pipeline{
6361
DMC_USER = credentials('MLBUILD_USER')
6462
DMC_PASSWORD = credentials('MLBUILD_PASSWORD')
6563
}
66-
stages{
67-
stage('tests'){
68-
steps{
64+
stages {
65+
stage('pull-request-tests') {
66+
when {
67+
not {
68+
expression {return params.regressions}
69+
}
70+
}
71+
steps {
6972
copyRPM 'Latest','11.0'
7073
setUpML '$WORKSPACE/xdmp/src/Mark*.rpm'
7174
copyConvertersRPM 'Latest','11.0'
7275
setUpMLConverters '$WORKSPACE/xdmp/src/Mark*Converters*.rpm'
73-
sh label:'deploy project', script: '''#!/bin/bash
76+
sh label:'deploy test app', script: '''#!/bin/bash
7477
export JAVA_HOME=$JAVA_HOME_DIR
7578
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
7679
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
7780
cd java-client-api
7881
./gradlew -i mlDeploy -PmlForestDataDirectory=/space
7982
'''
80-
sh label:'marklogic client test', script: '''#!/bin/bash
83+
sh label:'run marklogic-client-api tests', script: '''#!/bin/bash
8184
export JAVA_HOME=$JAVA_HOME_DIR
8285
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
8386
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
8487
cd java-client-api
85-
./gradlew marklogic-client-api:test || true
86-
'''
87-
sh label:'ml development tool test', script: '''#!/bin/bash
88-
export JAVA_HOME=$JAVA_HOME_DIR
89-
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
90-
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
91-
cd java-client-api
92-
./gradlew ml-development-tools:setupTestServer || true
93-
./gradlew ml-development-tools:generateTests || true
94-
./gradlew ml-development-tools:test || true
95-
'''
96-
sh '''
97-
cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/
98-
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
99-
cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/
100-
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
88+
./gradlew -i marklogic-client-api:test || true
10189
'''
10290
junit '**/build/**/TEST*.xml'
10391
}
@@ -111,37 +99,41 @@ pipeline{
11199
}
112100
}
113101
}
114-
stage('functional-tests'){
115-
when{
116-
allOf{
117-
branch 'develop'
118-
expression {return params.regressions}
119-
}
102+
103+
stage('regressions-11.0-Latest') {
104+
when {
105+
allOf {
106+
branch 'develop'
107+
expression {return params.regressions}
120108
}
121-
steps{
122-
copyRPM 'Latest','11.0'
123-
setUpML '$WORKSPACE/xdmp/src/Mark*.rpm'
124-
copyConvertersRPM 'Latest','11.0'
125-
setUpMLConverters '$WORKSPACE/xdmp/src/Mark*Converters*.rpm'
126-
sh label:'test', script: '''#!/bin/bash
127-
export JAVA_HOME=$JAVA_HOME_DIR
128-
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
129-
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
130-
cd java-client-api
131-
./gradlew -i mlDeploy -PmlForestDataDirectory=/space
132-
./gradlew marklogic-client-api-functionaltests:test || true
133-
'''
134-
sh '''
135-
cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/test/
136-
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
137-
'''
138-
junit '**/marklogic-client-api-functionaltests/**/build/**/TEST*.xml'
109+
}
110+
steps {
111+
runtests('Latest','11.0')
112+
junit '**/build/**/TEST*.xml'
113+
}
114+
post {
115+
unsuccessful {
116+
sendMail params.Email,'<h3>Some Tests Failed on Released 11.0 ML Nightly Server Single Node </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/java-client-api-regression/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'${STAGE_NAME} on develop against ML 11.0-nightly Failed'
139117
}
140-
post{
141-
unsuccessful{
142-
sendMail params.Email,'<h3>Some Tests Failed on Released 11.0 ML Nightly Server Single Node </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/java-client-api-regression/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'${STAGE_NAME} on develop against ML 11.0-nightly Failed'
143-
}
118+
}
119+
}
120+
121+
stage('regressions-10.0-9') {
122+
when {
123+
allOf {
124+
branch 'develop'
125+
expression {return params.regressions}
126+
}
127+
}
128+
steps {
129+
runtests('Release','10.0-9.5')
130+
junit '**/build/**/TEST*.xml'
131+
}
132+
post {
133+
unsuccessful {
134+
sendMail params.Email,'<h3>Some Tests Failed on Released 10.0-9.5 ML Server Single Node </h3><h4><a href=${JENKINS_URL}/blue/organizations/jenkins/java-client-api-regression/detail/$JOB_BASE_NAME/$BUILD_ID/tests><font color=red>Check the Test Report</font></a></h4><h4><a href=${RUN_DISPLAY_URL}>Check the Pipeline View</a></h4><h4> <a href=${BUILD_URL}/console> Check Console Output Here</a></h4><h4>Please create bugs for the failed regressions and fix them</h4>',false,'${STAGE_NAME} on develop against ML 10.0-9.5 Failed'
144135
}
136+
}
145137
}
146138
}
147139
}

‎marklogic-client-api-functionaltests/build.gradle‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,33 @@ dependencies {
3636
testImplementation group: 'ch.qos.logback', name: 'logback-classic', version:'1.2.11'
3737
}
3838

39-
task testFastFunctionalTests(type: Test) {
39+
task runFragileTests(type: Test) {
40+
description = "These are called 'fragile' because they'll pass when run by themselves, but when run as part of the " +
41+
"full suite, there seem to be one or more other fast functional tests that run before them and cause some of " +
42+
"their test methods to break. The Jenkinsfile thus calls these first before running the other functional " +
43+
"tests."
44+
include "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class"
45+
include "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class"
46+
include "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class"
47+
}
48+
49+
task runFastFunctionalTests(type: Test) {
4050
description = "Run all fast functional tests that don't setup/teardown custom app servers / databases"
4151
include "com/marklogic/client/fastfunctest/**"
52+
// Exclude the "fragile" ones
53+
exclude "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class"
54+
exclude "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class"
55+
exclude "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class"
4256
}
4357

44-
task testSlowFunctionalTests(type: Test) {
58+
task runSlowFunctionalTests(type: Test) {
4559
description = "Run slow functional tests; i.e. those that setup/teardown custom app servers / databases"
4660
include "com/marklogic/client/datamovement/functionaltests/**"
4761
include "com/marklogic/client/functionaltest/**"
48-
}
62+
}
63+
64+
task runFunctionalTests {
65+
dependsOn(runFragileTests, runFastFunctionalTests, runSlowFunctionalTests)
66+
}
67+
runFastFunctionalTests.mustRunAfter runFragileTests
68+
runSlowFunctionalTests.mustRunAfter runFastFunctionalTests

0 commit comments

Comments
 (0)