Skip to content

Commit da550fa

Browse files
sp-yducksp-yduck
committed
vmid scheduler
1 parent 99371ae commit da550fa

File tree

13 files changed

+201
-33
lines changed

13 files changed

+201
-33
lines changed

‎cloud/interfaces.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ type MachineGetter interface {
4949
GetScheduler(client *proxmox.Service) *scheduler.Scheduler
5050
Name() string
5151
Namespace() string
52+
Annotations() map[string]string
5253
// Zone() string
5354
// Role() string
5455
// IsControlPlane() bool

‎cloud/scheduler/framework/interface.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,8 @@ type NodeScorePlugin interface {
2020
Plugin
2121
Score(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nodeInfo *NodeInfo) (int64, *Status)
2222
}
23+
24+
type VMIDPlugin interface {
25+
Plugin
26+
Select(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions) (int, error)
27+
}

‎cloud/scheduler/framework/utils.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package framework
2+
3+
import "context"
4+
5+
type CtxKey string
6+
7+
// bind map's key-value to context key-value
8+
func ContextWithMap(ctx context.Context, m map[string]string) context.Context {
9+
for key, value := range m {
10+
ctx = context.WithValue(ctx, CtxKey(key), value)
11+
}
12+
return ctx
13+
}

‎cloud/scheduler/plugins/node/names/names.go‎ renamed to ‎cloud/scheduler/plugins/names/names.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,8 @@ const (
99
// score plugins
1010

1111
Random = "Random"
12+
13+
// vmid plugins
14+
15+
NextID = "NextID"
1216
)

‎cloud/scheduler/plugins/nextid/nextid.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package nextid
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"regexp"
7+
8+
"github.com/sp-yduck/proxmox-go/api"
9+
"github.com/sp-yduck/proxmox-go/proxmox"
10+
11+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
12+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
13+
)
14+
15+
type NextID struct {
16+
client *proxmox.Service
17+
}
18+
19+
var _ framework.VMIDPlugin = &NextID{}
20+
21+
const (
22+
Name = names.NextID
23+
VMIDRegexKey = "vmid.qemu-scheduler/regex"
24+
)
25+
26+
func New(client *proxmox.Service) *NextID {
27+
return &NextID{client: client}
28+
}
29+
30+
func (pl *NextID) Name() string {
31+
return Name
32+
}
33+
34+
// select minimum id being not used and matching regex
35+
// regex is specified in ctx value (key=vmid.qemu-scheduler/regex)
36+
func (pl *NextID) Select(ctx context.Context, _ *framework.CycleState, _ api.VirtualMachineCreateOptions) (int, error) {
37+
nextid, err := pl.client.RESTClient().GetNextID(ctx)
38+
if err != nil {
39+
return 0, err
40+
}
41+
idrange, err := findVMIDRange(ctx)
42+
if err != nil {
43+
return nextid, nil
44+
}
45+
table, err := pl.usedIDMap(ctx)
46+
if err != nil {
47+
return 0, err
48+
}
49+
for i := nextid; i < 1000000000; i++ {
50+
_, used := table[i]
51+
if idrange.MatchString(fmt.Sprintf("%d", i)) && !used {
52+
return i, nil
53+
}
54+
}
55+
return 0, fmt.Errorf("no available vmid")
56+
}
57+
58+
// specify available vmid as regex
59+
// example: vmid.qemu-scheduler/regex=(12[0-9]|130)
60+
func findVMIDRange(ctx context.Context) (*regexp.Regexp, error) {
61+
value := ctx.Value(framework.CtxKey(VMIDRegexKey))
62+
if value == nil {
63+
return nil, fmt.Errorf("no vmid range is specified")
64+
}
65+
reg, err := regexp.Compile(fmt.Sprintf("%s", value))
66+
if err != nil {
67+
return nil, err
68+
}
69+
return reg, nil
70+
}
71+
72+
// return map[vmid]bool
73+
func (pl *NextID) usedIDMap(ctx context.Context) (map[int]bool, error) {
74+
vms, err := pl.client.VirtualMachines(ctx)
75+
if err != nil {
76+
return nil, err
77+
}
78+
result := make(map[int]bool)
79+
for _, vm := range vms {
80+
result[vm.VMID] = true
81+
}
82+
return result, nil
83+
}

‎cloud/scheduler/plugins/node/nodename/node_name.go‎ renamed to ‎cloud/scheduler/plugins/nodename/node_name.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"github.com/sp-yduck/proxmox-go/api"
77

88
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
9-
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/node/names"
9+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
1010
)
1111

1212
type NodeName struct{}

‎cloud/scheduler/plugins/node/random/random.go‎ renamed to ‎cloud/scheduler/plugins/random/random.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/sp-yduck/proxmox-go/api"
99

1010
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
11-
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/node/names"
11+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
1212
)
1313

1414
type Random struct{}

‎cloud/scheduler/plugins/registry.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,36 @@
11
package plugins
22

33
import (
4+
"fmt"
5+
6+
"github.com/sp-yduck/proxmox-go/proxmox"
7+
48
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
5-
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/node/nodename"
6-
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/node/random"
9+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
10+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/nextid"
11+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/nodename"
12+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/random"
713
)
814

15+
func VMIDPlugins(client *proxmox.Service) map[string]framework.VMIDPlugin {
16+
return map[string]framework.VMIDPlugin{
17+
names.NextID: nextid.New(client),
18+
}
19+
}
20+
921
func NewNodeFilterPlugins() []framework.NodeFilterPlugin {
1022
return []framework.NodeFilterPlugin{&nodename.NodeName{}}
1123
}
1224

1325
func NewNodeScorePlugins() []framework.NodeScorePlugin {
1426
return []framework.NodeScorePlugin{&random.Random{}}
1527
}
28+
29+
func NewVMIDPlugin(client *proxmox.Service, name string) (framework.VMIDPlugin, error) {
30+
plugins := VMIDPlugins(client)
31+
plugin, ok := plugins[name]
32+
if !ok {
33+
return nil, fmt.Errorf("vmid plugin %s not found", name)
34+
}
35+
return plugin, nil
36+
}

‎cloud/scheduler/scheduler.go‎

Lines changed: 49 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,37 +30,68 @@ func NewManager(params SchedulerParams) *Manager {
3030

3131
func (m *Manager) NewScheduler(client *proxmox.Service) *Scheduler {
3232
logger := m.params.Logger.WithName("[qemu-scheduler]")
33-
nodeScheduler := NodeScheduler{filterPlugins: plugins.NewNodeFilterPlugins(), scorePlugins: plugins.NewNodeScorePlugins()}
34-
return &Scheduler{client: client, nodeScheduler: nodeScheduler, logger: logger}
33+
return &Scheduler{client: client, logger: logger}
3534
}
3635

3736
type Scheduler struct {
38-
client *proxmox.Service
39-
nodeScheduler NodeScheduler
40-
vmidScheduler VMIDScheduler
41-
logger logr.Logger
37+
client *proxmox.Service
38+
logger logr.Logger
4239
}
4340

4441
type SchedulerParams struct {
4542
Logger logr.Logger
4643
}
4744

4845
type NodeScheduler struct {
46+
client *proxmox.Service
4947
filterPlugins []framework.NodeFilterPlugin
5048
scorePlugins []framework.NodeScorePlugin
49+
logger logr.Logger
5150
}
5251

5352
type VMIDScheduler struct {
53+
client *proxmox.Service
54+
plugin framework.VMIDPlugin
55+
logger logr.Logger
5456
}
5557

56-
// just poc codes
57-
// return nextID fetched from Proxmox rest API nextID endpoint
58-
func (s *Scheduler) GetID(ctx context.Context) (int, error) {
59-
return s.client.RESTClient().GetNextID(ctx)
58+
func (s *Scheduler) NewNodeScheduler() NodeScheduler {
59+
return NodeScheduler{
60+
client: s.client,
61+
filterPlugins: plugins.NewNodeFilterPlugins(),
62+
scorePlugins: plugins.NewNodeScorePlugins(),
63+
logger: s.logger.WithValues("qemu-scheduler", "node"),
64+
}
65+
}
66+
67+
func (s *Scheduler) NewVMIDScheduler(name string) (VMIDScheduler, error) {
68+
plugin, err := plugins.NewVMIDPlugin(s.client, name)
69+
if err != nil {
70+
return VMIDScheduler{}, err
71+
}
72+
return VMIDScheduler{
73+
client: s.client,
74+
plugin: plugin,
75+
logger: s.logger.WithValues("qemu-scheduler", "vmid"),
76+
}, nil
6077
}
6178

6279
func (s *Scheduler) SelectNode(ctx context.Context, config api.VirtualMachineCreateOptions) (string, error) {
6380
s.logger.Info("finding proxmox node matching qemu")
81+
sched := s.NewNodeScheduler()
82+
return sched.run(ctx, config)
83+
}
84+
85+
func (s *Scheduler) SelectVMID(ctx context.Context, config api.VirtualMachineCreateOptions) (int, error) {
86+
s.logger.Info("finding proxmox vmid to be assigned to qemu")
87+
sched, err := s.NewVMIDScheduler("NextID")
88+
if err != nil {
89+
return 0, err
90+
}
91+
return sched.run(ctx, config)
92+
}
93+
94+
func (s *NodeScheduler) run(ctx context.Context, config api.VirtualMachineCreateOptions) (string, error) {
6495
nodes, err := s.client.Nodes(ctx)
6596
if err != nil {
6697
return "", err
@@ -88,7 +119,7 @@ func (s *Scheduler) SelectNode(ctx context.Context, config api.VirtualMachineCre
88119
return selectedNode, nil
89120
}
90121

91-
func (s *Scheduler) RunFilterPlugins(ctx context.Context, state *framework.CycleState, config api.VirtualMachineCreateOptions, nodes []*api.Node) ([]*api.Node, error) {
122+
func (s *NodeScheduler) RunFilterPlugins(ctx context.Context, state *framework.CycleState, config api.VirtualMachineCreateOptions, nodes []*api.Node) ([]*api.Node, error) {
92123
s.logger.Info("filtering proxmox node")
93124
feasibleNodes := make([]*api.Node, 0, len(nodes))
94125
nodeInfos, err := framework.GetNodeInfoList(ctx, s.client)
@@ -97,7 +128,7 @@ func (s *Scheduler) RunFilterPlugins(ctx context.Context, state *framework.Cycle
97128
}
98129
for _, nodeInfo := range nodeInfos {
99130
status := framework.NewStatus()
100-
for _, pl := range s.nodeScheduler.filterPlugins {
131+
for _, pl := range s.filterPlugins {
101132
status = pl.Filter(ctx, state, config, nodeInfo)
102133
if !status.IsSuccess() {
103134
status.SetFailedPlugin(pl.Name())
@@ -111,7 +142,7 @@ func (s *Scheduler) RunFilterPlugins(ctx context.Context, state *framework.Cycle
111142
return feasibleNodes, nil
112143
}
113144

114-
func (s *Scheduler) RunScorePlugins(ctx context.Context, state *framework.CycleState, config api.VirtualMachineCreateOptions, nodes []*api.Node) (framework.NodeScoreList, *framework.Status) {
145+
func (s *NodeScheduler) RunScorePlugins(ctx context.Context, state *framework.CycleState, config api.VirtualMachineCreateOptions, nodes []*api.Node) (framework.NodeScoreList, *framework.Status) {
115146
s.logger.Info("scoring proxmox node")
116147
var scoresMap map[string](map[int]framework.NodeScore)
117148
nodeInfos, err := framework.GetNodeInfoList(ctx, s.client)
@@ -121,7 +152,7 @@ func (s *Scheduler) RunScorePlugins(ctx context.Context, state *framework.CycleS
121152
return nil, status
122153
}
123154
for index, nodeInfo := range nodeInfos {
124-
for _, pl := range s.nodeScheduler.scorePlugins {
155+
for _, pl := range s.scorePlugins {
125156
score, status := pl.Score(ctx, state, config, nodeInfo)
126157
if !status.IsSuccess() {
127158
return nil, status
@@ -154,3 +185,7 @@ func selectHighestScoreNode(scoreList framework.NodeScoreList) (string, error) {
154185
}
155186
return selectedScore.Name, nil
156187
}
188+
189+
func (s *VMIDScheduler) run(ctx context.Context, config api.VirtualMachineCreateOptions) (int, error) {
190+
return s.plugin.Select(ctx, nil, config)
191+
}

‎cloud/scope/machine.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,10 @@ func (m *MachineScope) Namespace() string {
106106
return m.ProxmoxMachine.Namespace
107107
}
108108

109+
func (m *MachineScope) Annotations() map[string]string {
110+
return m.ProxmoxMachine.Annotations
111+
}
112+
109113
func (m *MachineScope) NodeName() string {
110114
return m.ProxmoxMachine.Spec.Node
111115
}

0 commit comments

Comments
 (0)