Skip to content

Commit 9340bef

Browse files
pooknullegegunesinelpandzic
authored
K8SPSMDB-974: add sleep infinity for mongod pods (#1315)
* K8SPSMDB-974: add `sleep infinity` for mongod pods https://jira.percona.com/browse/K8SPSMDB-974 * fix `golangci-lint` warns and replace `context.TODO()` * fix tests * fix tests * fix tests --------- Co-authored-by: Ege Güneş <ege.gunes@percona.com> Co-authored-by: Inel Pandzic <inel.pandzic@percona.com>
1 parent f005fe4 commit 9340bef

File tree

136 files changed

+1031
-315
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

136 files changed

+1031
-315
lines changed

‎build/ps-entry.sh‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,15 @@ fi
4646
file_env() {
4747
local var="$1"
4848
local fileVar="${var}_FILE"
49-
local def="${2:-}"
50-
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
49+
local def="${2-}"
50+
if [ "${!var-}" ] && [ "${!fileVar-}" ]; then
5151
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
5252
exit 1
5353
fi
5454
local val="$def"
55-
if [ "${!var:-}" ]; then
55+
if [ "${!var-}" ]; then
5656
val="${!var}"
57-
elif [ "${!fileVar:-}" ]; then
57+
elif [ "${!fileVar-}" ]; then
5858
val="$(<"${!fileVar}")"
5959
fi
6060
export "$var"="$val"
@@ -245,7 +245,20 @@ _dbPath() {
245245
echo "$dbPath"
246246
}
247247

248+
is_manual_recovery() {
249+
recovery_file='/data/db/sleep-forever'
250+
if [ -f "${recovery_file}" ]; then
251+
echo "The $recovery_file file is detected, node is going to infinity loop"
252+
echo "If you want to exit from infinity loop you need to remove $recovery_file file"
253+
while [ -f "${recovery_file}" ]; do
254+
sleep 1
255+
done
256+
fi
257+
}
258+
248259
if [ "$originalArgOne" = 'mongod' ]; then
260+
is_manual_recovery
261+
249262
file_env 'MONGO_INITDB_ROOT_USERNAME'
250263
file_env 'MONGO_INITDB_ROOT_PASSWORD'
251264
# pre-check a few factors to see if it's even worth bothering with initdb

‎cmd/mongodb-healthcheck/main.go‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,15 @@ func main() {
6565
os.Exit(0)
6666
}
6767

68+
sleepForever, err := fileExists("/data/db/sleep-forever")
69+
if err != nil {
70+
log.Error(err, "check if sleep-forever file exists")
71+
os.Exit(1)
72+
}
73+
if sleepForever {
74+
os.Exit(0)
75+
}
76+
6877
cnf, err := db.NewConfig(
6978
app,
7079
pkg.EnvMongoDBClusterMonitorUser,
@@ -82,38 +91,22 @@ func main() {
8291
os.Exit(1)
8392
}
8493

85-
client, err := db.Dial(ctx, cnf)
86-
if err != nil {
87-
log.Error(err, "connection error")
88-
os.Exit(1)
89-
}
90-
91-
defer func() {
92-
if err := client.Disconnect(ctx); err != nil {
93-
log.Error(err, "failed to disconnect")
94-
os.Exit(1)
95-
}
96-
}()
97-
9894
switch command {
99-
10095
case "k8s liveness":
10196
log.Info("Running Kubernetes liveness check for", "component", component)
10297
switch *component {
10398

10499
case "mongod":
105-
memberState, err := healthcheck.HealthCheckMongodLiveness(client, int64(*startupDelaySeconds))
100+
memberState, err := healthcheck.HealthCheckMongodLiveness(ctx, cnf, int64(*startupDelaySeconds))
106101
if err != nil {
107-
client.Disconnect(ctx) // nolint:golint,errcheck
108102
log.Error(err, "Member failed Kubernetes liveness check")
109103
os.Exit(1)
110104
}
111105
log.Info("Member passed Kubernetes liveness check with replication state", "state", memberState)
112106

113107
case "mongos":
114-
err := healthcheck.HealthCheckMongosLiveness(client)
108+
err := healthcheck.HealthCheckMongosLiveness(ctx, cnf)
115109
if err != nil {
116-
client.Disconnect(ctx) // nolint:golint,errcheck
117110
log.Error(err, "Member failed Kubernetes liveness check")
118111
os.Exit(1)
119112
}
@@ -125,14 +118,14 @@ func main() {
125118
switch *component {
126119

127120
case "mongod":
128-
client.Disconnect(ctx) // nolint:golint,errcheck
129-
log.Error(err, "readiness check for mongod is not implemented")
130-
os.Exit(1)
131-
121+
err := healthcheck.MongodReadinessCheck(ctx, cnf.Hosts[0])
122+
if err != nil {
123+
log.Error(err, "Member failed Kubernetes readiness check")
124+
os.Exit(1)
125+
}
132126
case "mongos":
133-
err := healthcheck.MongosReadinessCheck(ctx, client)
127+
err := healthcheck.MongosReadinessCheck(ctx, cnf)
134128
if err != nil {
135-
client.Disconnect(ctx) // nolint:golint,errcheck
136129
log.Error(err, "Member failed Kubernetes readiness check")
137130
os.Exit(1)
138131
}

‎e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter-oc.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,17 @@ spec:
104104
name: mongodb
105105
protocol: TCP
106106
readinessProbe:
107+
exec:
108+
command:
109+
- /opt/percona/mongodb-healthcheck
110+
- k8s
111+
- readiness
112+
- --component
113+
- mongod
107114
failureThreshold: 8
108115
initialDelaySeconds: 10
109116
periodSeconds: 3
110117
successThreshold: 1
111-
tcpSocket:
112-
port: 27017
113118
timeoutSeconds: 2
114119
resources: {}
115120
securityContext:

‎e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,17 @@ spec:
9898
name: mongodb
9999
protocol: TCP
100100
readinessProbe:
101+
exec:
102+
command:
103+
- /opt/percona/mongodb-healthcheck
104+
- k8s
105+
- readiness
106+
- --component
107+
- mongod
101108
failureThreshold: 8
102109
initialDelaySeconds: 10
103110
periodSeconds: 3
104111
successThreshold: 1
105-
tcpSocket:
106-
port: 27017
107112
timeoutSeconds: 2
108113
resources: {}
109114
securityContext:

‎e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter-oc.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,17 @@ spec:
105105
name: mongodb
106106
protocol: TCP
107107
readinessProbe:
108+
exec:
109+
command:
110+
- /opt/percona/mongodb-healthcheck
111+
- k8s
112+
- readiness
113+
- --component
114+
- mongod
108115
failureThreshold: 8
109116
initialDelaySeconds: 10
110117
periodSeconds: 3
111118
successThreshold: 1
112-
tcpSocket:
113-
port: 27017
114119
timeoutSeconds: 2
115120
resources: {}
116121
securityContext:

‎e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,17 @@ spec:
105105
name: mongodb
106106
protocol: TCP
107107
readinessProbe:
108+
exec:
109+
command:
110+
- /opt/percona/mongodb-healthcheck
111+
- k8s
112+
- readiness
113+
- --component
114+
- mongod
108115
failureThreshold: 8
109116
initialDelaySeconds: 10
110117
periodSeconds: 3
111118
successThreshold: 1
112-
tcpSocket:
113-
port: 27017
114119
timeoutSeconds: 2
115120
resources: {}
116121
securityContext:

‎e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,17 @@ spec:
9292
name: mongodb
9393
protocol: TCP
9494
readinessProbe:
95+
exec:
96+
command:
97+
- /opt/percona/mongodb-healthcheck
98+
- k8s
99+
- readiness
100+
- --component
101+
- mongod
95102
failureThreshold: 3
96103
initialDelaySeconds: 10
97104
periodSeconds: 3
98105
successThreshold: 1
99-
tcpSocket:
100-
port: 27017
101106
timeoutSeconds: 2
102107
resources: {}
103108
securityContext:

‎e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,17 @@ spec:
9292
name: mongodb
9393
protocol: TCP
9494
readinessProbe:
95+
exec:
96+
command:
97+
- /opt/percona/mongodb-healthcheck
98+
- k8s
99+
- readiness
100+
- --component
101+
- mongod
95102
failureThreshold: 3
96103
initialDelaySeconds: 10
97104
periodSeconds: 3
98105
successThreshold: 1
99-
tcpSocket:
100-
port: 27017
101106
timeoutSeconds: 2
102107
resources: {}
103108
securityContext:

‎e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,17 @@ spec:
9393
name: mongodb
9494
protocol: TCP
9595
readinessProbe:
96+
exec:
97+
command:
98+
- /opt/percona/mongodb-healthcheck
99+
- k8s
100+
- readiness
101+
- --component
102+
- mongod
96103
failureThreshold: 8
97104
initialDelaySeconds: 10
98105
periodSeconds: 3
99106
successThreshold: 1
100-
tcpSocket:
101-
port: 27017
102107
timeoutSeconds: 2
103108
resources:
104109
limits:

‎e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,17 @@ spec:
9393
name: mongodb
9494
protocol: TCP
9595
readinessProbe:
96+
exec:
97+
command:
98+
- /opt/percona/mongodb-healthcheck
99+
- k8s
100+
- readiness
101+
- --component
102+
- mongod
96103
failureThreshold: 8
97104
initialDelaySeconds: 10
98105
periodSeconds: 3
99106
successThreshold: 1
100-
tcpSocket:
101-
port: 27017
102107
timeoutSeconds: 2
103108
resources:
104109
limits:

0 commit comments

Comments
 (0)