Skip to content

Commit 458143b

Browse files
committed
*: Save Xenon's metadata to config map.
subpath -> configmap subpath may not work in some conditions. 1. Add a Xenon Configmap which is used to save xenon data. 2. Build peers.json according to the replicas. 3. Make peers.json in init-sidecar. 4. Xenon boot using existing peers.json.
1 parent 8669092 commit 458143b

File tree

12 files changed

+194
-16
lines changed

12 files changed

+194
-16
lines changed

controllers/mysqlcluster_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
125125
clustersyncer.NewFollowerSVCSyncer(r.Client, instance),
126126
clustersyncer.NewStatefulSetSyncer(r.Client, instance, cmRev, sctRev, r.SQLRunnerFactory, r.XenonExecutor),
127127
clustersyncer.NewPDBSyncer(r.Client, instance),
128+
clustersyncer.NewXenonCMSyncer(r.Client, instance),
128129
}
129130

130131
if instance.Spec.MetricsOpts.Enabled {

mysqlcluster/container/init_sidecar.go

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,21 @@ func (c *initSidecar) getVolumeMounts() []corev1.VolumeMount {
170170
Name: utils.MysqlCMVolumeName,
171171
MountPath: utils.MysqlCMVolumeMountPath,
172172
},
173+
{
174+
Name: utils.XenonCMVolumeName,
175+
MountPath: utils.XenonCMVolumeMountPath,
176+
},
177+
{
178+
Name: utils.XenonMetaVolumeName,
179+
MountPath: utils.XenonMetaVolumeMountPath,
180+
},
173181
{
174182
Name: utils.ScriptsVolumeName,
175183
MountPath: utils.ScriptsVolumeMountPath,
176184
},
177185
{
178-
Name: utils.XenonVolumeName,
179-
MountPath: utils.XenonVolumeMountPath,
186+
Name: utils.XenonConfVolumeName,
187+
MountPath: utils.XenonConfVolumeMountPath,
180188
},
181189
{
182190
Name: utils.InitFileVolumeName,

mysqlcluster/container/init_sidecar_test.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -279,13 +279,21 @@ var (
279279
Name: utils.MysqlCMVolumeName,
280280
MountPath: utils.MysqlCMVolumeMountPath,
281281
},
282+
{
283+
Name: utils.XenonCMVolumeName,
284+
MountPath: utils.XenonCMVolumeMountPath,
285+
},
286+
{
287+
Name: utils.XenonMetaVolumeName,
288+
MountPath: utils.XenonMetaVolumeMountPath,
289+
},
282290
{
283291
Name: utils.ScriptsVolumeName,
284292
MountPath: utils.ScriptsVolumeMountPath,
285293
},
286294
{
287-
Name: utils.XenonVolumeName,
288-
MountPath: utils.XenonVolumeMountPath,
295+
Name: utils.XenonConfVolumeName,
296+
MountPath: utils.XenonConfVolumeMountPath,
289297
},
290298
{
291299
Name: utils.InitFileVolumeName,
@@ -437,7 +445,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
437445
MysqlCluster: &testToKuDBMysqlCluster,
438446
}
439447
tokudbCase := EnsureContainer("init-sidecar", &testTokuDBCluster)
440-
tokuDBVolumeMounts := make([]corev1.VolumeMount, 6, 7)
448+
tokuDBVolumeMounts := make([]corev1.VolumeMount, 8, 9)
441449
copy(tokuDBVolumeMounts, defaultInitsidecarVolumeMounts)
442450
tokuDBVolumeMounts = append(tokuDBVolumeMounts, corev1.VolumeMount{
443451
Name: utils.SysVolumeName,
@@ -453,7 +461,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
453461
MysqlCluster: &testPersistenceMysqlCluster,
454462
}
455463
persistenceCase := EnsureContainer("init-sidecar", &testPersistenceCluster)
456-
persistenceVolumeMounts := make([]corev1.VolumeMount, 6, 7)
464+
persistenceVolumeMounts := make([]corev1.VolumeMount, 8, 9)
457465
copy(persistenceVolumeMounts, defaultInitsidecarVolumeMounts)
458466
persistenceVolumeMounts = append(persistenceVolumeMounts, corev1.VolumeMount{
459467
Name: utils.DataVolumeName,

mysqlcluster/container/xenon.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,12 @@ func (c *xenon) getVolumeMounts() []corev1.VolumeMount {
115115
MountPath: utils.ScriptsVolumeMountPath,
116116
},
117117
{
118-
Name: utils.XenonVolumeName,
119-
MountPath: utils.XenonVolumeMountPath,
118+
Name: utils.XenonConfVolumeName,
119+
MountPath: utils.XenonConfVolumeMountPath,
120+
},
121+
{
122+
Name: utils.XenonMetaVolumeName,
123+
MountPath: utils.XenonMetaVolumeMountPath,
120124
},
121125
{
122126
Name: utils.SysLocalTimeZone,

mysqlcluster/container/xenon_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,13 @@ func TestGetXenonVolumeMounts(t *testing.T) {
135135
MountPath: "/scripts",
136136
},
137137
{
138-
Name: "xenon",
138+
Name: "xenon-conf",
139139
MountPath: "/etc/xenon",
140140
},
141+
{
142+
Name: "xenon-meta",
143+
MountPath: "/var/lib/xenon",
144+
},
141145
{
142146
Name: utils.SysLocalTimeZone,
143147
MountPath: "/etc/localtime",

mysqlcluster/mysqlcluster.go

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,30 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
214214
},
215215
},
216216
},
217+
corev1.Volume{
218+
Name: utils.XenonCMVolumeName,
219+
VolumeSource: corev1.VolumeSource{
220+
ConfigMap: &corev1.ConfigMapVolumeSource{
221+
LocalObjectReference: corev1.LocalObjectReference{
222+
Name: c.GetNameForResource(utils.XenonMetaData),
223+
},
224+
},
225+
},
226+
},
227+
corev1.Volume{
228+
Name: utils.XenonMetaVolumeName,
229+
VolumeSource: corev1.VolumeSource{
230+
EmptyDir: &corev1.EmptyDirVolumeSource{},
231+
},
232+
},
217233
corev1.Volume{
218234
Name: utils.ScriptsVolumeName,
219235
VolumeSource: corev1.VolumeSource{
220236
EmptyDir: &corev1.EmptyDirVolumeSource{},
221237
},
222238
},
223239
corev1.Volume{
224-
Name: utils.XenonVolumeName,
240+
Name: utils.XenonConfVolumeName,
225241
VolumeSource: corev1.VolumeSource{
226242
EmptyDir: &corev1.EmptyDirVolumeSource{},
227243
},
@@ -294,6 +310,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
294310
return fmt.Sprintf("%s-metrics", c.Name)
295311
case utils.Secret:
296312
return fmt.Sprintf("%s-secret", c.Name)
313+
case utils.XenonMetaData:
314+
return fmt.Sprintf("%s-xenon", c.Name)
297315
default:
298316
return c.Name
299317
}

mysqlcluster/mysqlcluster_test.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,14 +266,30 @@ func TestEnsureVolumes(t *testing.T) {
266266
},
267267
},
268268
},
269+
{
270+
Name: utils.XenonCMVolumeName,
271+
VolumeSource: corev1.VolumeSource{
272+
ConfigMap: &corev1.ConfigMapVolumeSource{
273+
LocalObjectReference: corev1.LocalObjectReference{
274+
Name: "sample-xenon",
275+
},
276+
},
277+
},
278+
},
279+
{
280+
Name: utils.XenonMetaVolumeName,
281+
VolumeSource: corev1.VolumeSource{
282+
EmptyDir: &corev1.EmptyDirVolumeSource{},
283+
},
284+
},
269285
{
270286
Name: utils.ScriptsVolumeName,
271287
VolumeSource: corev1.VolumeSource{
272288
EmptyDir: &corev1.EmptyDirVolumeSource{},
273289
},
274290
},
275291
{
276-
Name: utils.XenonVolumeName,
292+
Name: utils.XenonConfVolumeName,
277293
VolumeSource: corev1.VolumeSource{
278294
EmptyDir: &corev1.EmptyDirVolumeSource{},
279295
},

mysqlcluster/syncer/xenon_cm.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
Copyright 2021 RadonDB.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package syncer
18+
19+
import (
20+
"encoding/json"
21+
"fmt"
22+
23+
"github.com/presslabs/controller-util/syncer"
24+
corev1 "k8s.io/api/core/v1"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"sigs.k8s.io/controller-runtime/pkg/client"
27+
28+
"github.com/radondb/radondb-mysql-kubernetes/mysqlcluster"
29+
"github.com/radondb/radondb-mysql-kubernetes/utils"
30+
)
31+
32+
// NewXenonCMSyncer returns xenon configmap syncer.
33+
func NewXenonCMSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
34+
cm := &corev1.ConfigMap{
35+
TypeMeta: metav1.TypeMeta{
36+
APIVersion: "v1",
37+
Kind: "ConfigMap",
38+
},
39+
ObjectMeta: metav1.ObjectMeta{
40+
Name: c.GetNameForResource(utils.XenonMetaData),
41+
Namespace: c.Namespace,
42+
Labels: c.GetLabels(),
43+
},
44+
}
45+
46+
return syncer.NewObjectSyncer("ConfigMap", c.Unwrap(), cm, cli, func() error {
47+
if int(*c.Spec.Replicas) == 0 {
48+
return nil
49+
}
50+
51+
metaData, err := buildXenonMetaData(c)
52+
if err != nil {
53+
return fmt.Errorf("failed to build xenon metadata: %s", err)
54+
}
55+
56+
cm.Data = map[string]string{
57+
"peers.json": metaData,
58+
}
59+
60+
return nil
61+
})
62+
}
63+
64+
type XenonMetaData struct {
65+
Idlepeers []string `json:"idlepeers"`
66+
Peers []string `json:"peers"`
67+
}
68+
69+
// buildXenonMetaData build the default metadata of xenon.
70+
func buildXenonMetaData(c *mysqlcluster.MysqlCluster) (string, error) {
71+
replicas := c.Spec.Replicas
72+
xenonMetaData := XenonMetaData{}
73+
for i := 0; i < int(*replicas); i++ {
74+
xenonMetaData.Peers = append(xenonMetaData.Peers,
75+
fmt.Sprintf("%s-%d.%s.%s:%d",
76+
c.GetNameForResource(utils.StatefulSet),
77+
i,
78+
c.GetNameForResource(utils.HeadlessSVC),
79+
c.Namespace,
80+
utils.XenonPort,
81+
))
82+
}
83+
metaJson, err := json.Marshal(xenonMetaData)
84+
if err != nil {
85+
return "", err
86+
}
87+
return string(metaJson), nil
88+
}

sidecar/config.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ func (cfg *Config) buildXenonConf() []byte {
344344
"election-timeout": %d,
345345
"admit-defeat-hearbeat-count": %d,
346346
"heartbeat-timeout": %d,
347-
"meta-datadir": "/var/lib/xenon/",
347+
"meta-datadir": "%s",
348348
"semi-sync-degrade": true,
349349
"purge-binlog-disabled": true,
350350
"super-idle": false
@@ -353,7 +353,7 @@ func (cfg *Config) buildXenonConf() []byte {
353353
`, hostName, utils.XenonPort, hostName, utils.XenonPeerPort, cfg.ReplicationPassword, cfg.ReplicationUser,
354354
cfg.GtidPurged, requestTimeout,
355355
pingTimeout, cfg.RootPassword, version, srcSysVars, replicaSysVars, cfg.ElectionTimeout,
356-
cfg.AdmitDefeatHearbeatCount, heartbeatTimeout)
356+
cfg.AdmitDefeatHearbeatCount, heartbeatTimeout, xenonConfigPath)
357357

358358
return utils.StringToBytes(str)
359359
}

sidecar/init.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ func runInitCommand(cfg *Config) error {
141141
return fmt.Errorf("failed to copy my.cnf: %s", err)
142142
}
143143

144+
buildDefaultXenonMeta(uid, gid)
145+
144146
// build client.conf.
145147
clientConfig, err := cfg.buildClientConfig()
146148
if err != nil {
@@ -290,3 +292,21 @@ chown -R mysql.mysql %s`,
290292
// chown -R mysql.mysql extra.cnf
291293
path.Join(extraConfPath, "extra.cnf"))
292294
}
295+
296+
func buildDefaultXenonMeta(uid, gid int) error {
297+
metaFile := fmt.Sprintf("%s/peers.json", xenonConfigPath)
298+
// mkdir var/lib/xenon.
299+
// https://github.com/radondb/xenon/blob/master/src/raft/raft.go#L118
300+
if err := os.MkdirAll(xenonConfigPath, 0777); err != nil {
301+
return fmt.Errorf("failed to mkdir %s: %s", xenonConfigPath, err)
302+
}
303+
// copy appropriate peers.json from config-map to config mount.
304+
if err := copyFile(path.Join(xenonCMPath, "peers.json"), path.Join(xenonConfigPath, "peers.json")); err != nil {
305+
return fmt.Errorf("failed to copy peers.json: %s", err)
306+
}
307+
// chown -R mysql:mysql /var/lib/xenon/peers.json.
308+
if err := os.Chown(metaFile, uid, gid); err != nil {
309+
return fmt.Errorf("failed to chown %s: %s", metaFile, err)
310+
}
311+
return nil
312+
}

0 commit comments

Comments
 (0)