Skip to content

Commit 53871be

Browse files
ajanikowajanikow
authored
[Feature] Advanced InSync discovery (#925)
1 parent 31fa8fb commit 53871be

33 files changed

+28349
-348
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
- (Bugfix) Assign imagePullSecrets to LocalStorage
88
- (Update) Bump K8S API to 1.21.10
99
- (Feature) (ACS) Add ACS handler
10+
- (Feature) Allow to restart DBServers in cases when WriteConcern will be satisfied
1011

1112
## [1.2.8](https://github.com/arangodb/kube-arangodb/tree/1.2.8) (2022-02-24)
1213
- Do not check License V2 on Community images

‎pkg/deployment/agency/current_collections.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,5 @@ type StateCurrentDBCollections map[string]StateCurrentDBCollection
2727
type StateCurrentDBCollection map[string]StateCurrentDBShard
2828

2929
type StateCurrentDBShard struct {
30-
Servers []string `json:"servers,omitempty"`
30+
Servers ShardServers `json:"servers,omitempty"`
3131
}

‎pkg/deployment/agency/generator_collection_test.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
21+
package agency
22+
23+
type CollectionGeneratorInterface interface {
24+
WithWriteConcern(wc int) CollectionGeneratorInterface
25+
WithReplicationFactor(rf int) CollectionGeneratorInterface
26+
WithShard() ShardGeneratorInterface
27+
Add() DatabaseGeneratorInterface
28+
}
29+
30+
type collectionGenerator struct {
31+
db databaseGenerator
32+
col string
33+
34+
wc *int
35+
rf *int
36+
shards map[int]shardGenerator
37+
}
38+
39+
func (c collectionGenerator) Add() DatabaseGeneratorInterface {
40+
d := c.db
41+
if d.collections == nil {
42+
d.collections = map[string]collectionGenerator{}
43+
}
44+
45+
d.collections[c.col] = c
46+
47+
return d
48+
}
49+
50+
func (c collectionGenerator) WithShard() ShardGeneratorInterface {
51+
return shardGenerator{
52+
col: c,
53+
}
54+
}
55+
56+
func (c collectionGenerator) WithWriteConcern(wc int) CollectionGeneratorInterface {
57+
c.wc = &wc
58+
return c
59+
}
60+
61+
func (c collectionGenerator) WithReplicationFactor(rf int) CollectionGeneratorInterface {
62+
c.rf = &rf
63+
return c
64+
}

‎pkg/deployment/agency/generator_database_test.go‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
21+
package agency
22+
23+
import (
24+
"fmt"
25+
"strings"
26+
"testing"
27+
28+
"github.com/arangodb/kube-arangodb/pkg/util"
29+
"github.com/dchest/uniuri"
30+
"github.com/stretchr/testify/require"
31+
)
32+
33+
func NewDatabaseRandomGenerator() DatabaseGeneratorInterface {
34+
return NewDatabaseGenerator(fmt.Sprintf("d%s", strings.ToLower(uniuri.NewLen(16))))
35+
}
36+
37+
func NewDatabaseGenerator(name string) DatabaseGeneratorInterface {
38+
return databaseGenerator{
39+
db: name,
40+
}
41+
}
42+
43+
type DatabaseGeneratorInterface interface {
44+
Collection(name string) CollectionGeneratorInterface
45+
RandomCollection() CollectionGeneratorInterface
46+
Add() StateGenerator
47+
}
48+
49+
type databaseGenerator struct {
50+
db string
51+
52+
collections map[string]collectionGenerator
53+
}
54+
55+
func (d databaseGenerator) RandomCollection() CollectionGeneratorInterface {
56+
return d.Collection(fmt.Sprintf("c%s", strings.ToLower(uniuri.NewLen(16))))
57+
}
58+
59+
func (d databaseGenerator) Collection(name string) CollectionGeneratorInterface {
60+
return collectionGenerator{
61+
db: d,
62+
col: name,
63+
}
64+
}
65+
66+
func (d databaseGenerator) Add() StateGenerator {
67+
return func(t *testing.T, s *State) {
68+
if s.Plan.Collections == nil {
69+
s.Plan.Collections = StatePlanCollections{}
70+
}
71+
72+
if s.Current.Collections == nil {
73+
s.Current.Collections = StateCurrentCollections{}
74+
}
75+
76+
_, ok := s.Plan.Collections[d.db]
77+
require.False(t, ok)
78+
79+
_, ok = s.Current.Collections[d.db]
80+
require.False(t, ok)
81+
82+
plan := StatePlanDBCollections{}
83+
current := StateCurrentDBCollections{}
84+
85+
for col, colDet := range d.collections {
86+
planShards := Shards{}
87+
currentShards := StateCurrentDBCollection{}
88+
89+
for shard, shardDet := range colDet.shards {
90+
n := fmt.Sprintf("s%d", shard)
91+
92+
planShards[n] = shardDet.plan
93+
currentShards[n] = StateCurrentDBShard{Servers: shardDet.current}
94+
}
95+
96+
planCol := StatePlanCollection{
97+
Name: util.NewString(col),
98+
Shards: planShards,
99+
WriteConcern: colDet.wc,
100+
ReplicationFactor: colDet.rf,
101+
}
102+
103+
plan[col] = planCol
104+
current[col] = currentShards
105+
}
106+
107+
s.Plan.Collections[d.db] = plan
108+
s.Current.Collections[d.db] = current
109+
}
110+
}

‎pkg/deployment/agency/generator_shard_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
21+
package agency
22+
23+
type ShardGeneratorInterface interface {
24+
WithPlan(servers ...string) ShardGeneratorInterface
25+
WithCurrent(servers ...string) ShardGeneratorInterface
26+
Add() CollectionGeneratorInterface
27+
}
28+
29+
type shardGenerator struct {
30+
col collectionGenerator
31+
32+
plan []string
33+
current []string
34+
}
35+
36+
func (s shardGenerator) WithPlan(servers ...string) ShardGeneratorInterface {
37+
s.plan = servers
38+
return s
39+
}
40+
41+
func (s shardGenerator) WithCurrent(servers ...string) ShardGeneratorInterface {
42+
s.current = servers
43+
return s
44+
}
45+
46+
func (s shardGenerator) Add() CollectionGeneratorInterface {
47+
c := s.col
48+
49+
if c.shards == nil {
50+
c.shards = map[int]shardGenerator{}
51+
}
52+
53+
c.shards[id()] = s
54+
55+
return c
56+
}

‎pkg/deployment/agency/generator_test.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
21+
package agency
22+
23+
import (
24+
"sync"
25+
"testing"
26+
)
27+
28+
var (
29+
currentID int
30+
idLock sync.Mutex
31+
)
32+
33+
func id() int {
34+
idLock.Lock()
35+
defer idLock.Unlock()
36+
37+
z := currentID
38+
currentID++
39+
return z
40+
}
41+
42+
type StateGenerator func(t *testing.T, s *State)
43+
44+
func GenerateState(t *testing.T, generators ...StateGenerator) State {
45+
var s State
46+
47+
for _, g := range generators {
48+
g(t, &s)
49+
}
50+
51+
return s
52+
}

‎pkg/deployment/agency/plan_collections.go‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,62 @@ func (a StatePlanDBCollections) IsDBServerInCollections(name string) bool {
4242
return false
4343
}
4444

45+
func (a StatePlanDBCollections) CountShards() int {
46+
count := 0
47+
48+
for _, d := range a {
49+
count += len(d.Shards)
50+
}
51+
52+
return count
53+
}
54+
4555
type StatePlanCollection struct {
46-
Name *string `json:"name"`
47-
Shards StatePlanShard `json:"shards"`
56+
Name *string `json:"name"`
57+
Shards Shards `json:"shards"`
58+
// deprecated
59+
// MinReplicationFactor is deprecated, but we have to support it for backward compatibility
60+
MinReplicationFactor *int `json:"minReplicationFactor,omitempty"`
61+
WriteConcern *int `json:"writeConcern,omitempty"`
62+
ReplicationFactor *int `json:"replicationFactor,omitempty"`
63+
}
64+
65+
func (a *StatePlanCollection) GetReplicationFactor(shard string) int {
66+
if a == nil {
67+
return 0
68+
}
69+
70+
l := len(a.Shards[shard])
71+
72+
if z := a.ReplicationFactor; z == nil {
73+
return l
74+
} else {
75+
if v := *z; v > l {
76+
return v
77+
} else {
78+
return l
79+
}
80+
}
81+
}
82+
83+
func (a *StatePlanCollection) GetWriteConcern(def int) int {
84+
if p := a.GetWriteConcernP(); p != nil {
85+
return *p
86+
}
87+
88+
return def
89+
}
90+
91+
func (a *StatePlanCollection) GetWriteConcernP() *int {
92+
if a == nil {
93+
return nil
94+
}
95+
96+
if a.WriteConcern == nil {
97+
return a.MinReplicationFactor
98+
}
99+
100+
return a.WriteConcern
48101
}
49102

50103
func (a StatePlanCollection) GetName(d string) string {
@@ -55,15 +108,15 @@ func (a StatePlanCollection) GetName(d string) string {
55108
return *a.Name
56109
}
57110

58-
func (a StatePlanCollection) IsDBServerInShards(name string) bool {
59-
for _, dbservers := range a.Shards {
60-
for _, dbserver := range dbservers {
61-
if dbserver == name {
62-
return true
63-
}
111+
func (a *StatePlanCollection) IsDBServerInShards(name string) bool {
112+
if a == nil {
113+
return false
114+
}
115+
116+
for _, planShards := range a.Shards {
117+
if planShards.Contains(name) {
118+
return true
64119
}
65120
}
66121
return false
67122
}
68-
69-
type StatePlanShard map[string][]string

0 commit comments

Comments
 (0)