Skip to content

Commit 483062e

Browse files
authored
Merge pull request #102 from sp-yduck/feature/qemu-scheduler
new feature qemu-scheduler
2 parents 1df63fb + 84901fa commit 483062e

36 files changed

+1955
-87
lines changed

Makefile

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
5757
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
5858

5959
.PHONY: fmt
60-
fmt: ## Run go fmt against code.
60+
fmt: goimports ## Run go fmt against code.
6161
go fmt ./...
62+
$(GOIMPORTS) -w ./
6263

6364
.PHONY: vet
6465
vet: ## Run go vet against code.
@@ -232,6 +233,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
232233
ENVSUBST ?= $(LOCALBIN)/envsubst
233234
KUBECTL ?= $(LOCALBIN)/kubectl
234235
GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
236+
GOIMPORTS ?= $(LOCALBIN)/goimports
235237

236238
## Tool Versions
237239
KUSTOMIZE_VERSION ?= v5.0.0
@@ -280,3 +282,8 @@ $(SETUP_ENVTEST): go.mod # Build setup-envtest from tools folder.
280282
golangci-lint: $(GOLANGCI_LINT)
281283
$(GOLANGCI_LINT): $(LOCALBIN)
282284
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LOCALBIN) v1.54.0
285+
286+
.PHONY: goimports
287+
goimports: $(GOIMPORTS)
288+
$(GOIMPORTS): $(LOCALBIN)
289+
GOBIN=$(LOCALBIN) go install golang.org/x/tools/cmd/goimports@latest

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ kubectl delete cluster cappx-test
6363

6464
- Supports custom cloud-config (user data). CAPPX uses VNC websockert for bootstrapping nodes so it can applies custom cloud-config that can not be achieved by only Proxmox API.
6565

66+
- Flexible vmid/node assigning. You can flexibly assign vmid to your qemu and flexibly schedule qemus to proxmox nodes. For more details please check [qemu-scheduler](./cloud/scheduler/).
67+
6668
### Node Images
6769

6870
CAPPX is compatible with `iso`, `qcow2`, `qed`, `raw`, `vdi`, `vpc`, `vmdk` format of image. You can build your own node image and use it for `ProxmoxMachine`.

cloud/interfaces.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
99

1010
infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
11+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler"
1112
)
1213

1314
type Reconciler interface {
@@ -45,8 +46,10 @@ type ClusterSettter interface {
4546
// MachineGetter is an interface which can get machine information.
4647
type MachineGetter interface {
4748
Client
49+
GetScheduler(client *proxmox.Service) *scheduler.Scheduler
4850
Name() string
4951
Namespace() string
52+
Annotations() map[string]string
5053
// Zone() string
5154
// Role() string
5255
// IsControlPlane() bool

cloud/scheduler/README.md

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# qemu-scheduler
2+
3+
Scheduling refers to making sure that VM(QEMU) are matched to Proxmox Nodes.
4+
5+
## How qemu-scheduler select proxmox node to run qemu
6+
7+
Basic flow of the node selection process is `filter => score => select one node which has highest score`
8+
9+
### Filter Plugins
10+
11+
Filter plugins filter the node based on nodename, overcommit ratio etc.
12+
13+
#### regex plugin
14+
15+
Regex plugin is a one of the default Filter Plugin of qemu-scheduler. You can specify node name as regex format.
16+
```sh
17+
key: node.qemu-scheduler/regex
18+
value(example): node[0-9]+
19+
```
20+
21+
### Score Plugins
22+
23+
Score plugins score the nodes based on resource etc.
24+
25+
## How to specify vmid
26+
qemu-scheduler reads context and find key registerd to scheduler. If the context has any value of the registerd key, qemu-scheduler uses the plugin that matchies the key.
27+
28+
### Range Plugin
29+
You can specify vmid range with `(start id)-(end id)` format.
30+
```sh
31+
key: vmid.qemu-scheduler/range
32+
value(example): 100-150
33+
```
34+
35+
### Regex Plugin
36+
```sh
37+
key: vmid.qemu-scheduler/regex
38+
value(example): (12[0-9]|130)
39+
```
40+
41+
## How qemu-scheduler works with CAPPX
42+
CAPPX passes all the annotation (of `ProxmoxMachine`) key-values to scheduler's context. So if you will use Range Plugin for your `ProxmoxMachine`, your manifest must look like following.
43+
```sh
44+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
45+
kind: ProxmoxMachine
46+
metadata:
47+
name: sample-machine
48+
annotations:
49+
vmid.qemu-scheduler/range: 100-150 # this means your vmid will be chosen from the range of 100 to 150.
50+
```
51+
52+
Also, you can specifies these annotations via `MachineDeployment` since Cluster API propagates some metadatas (ref: [metadata-propagation](https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation.html#metadata-propagation)).
53+
54+
For example, your `MachineDeployment` may look like following.
55+
```sh
56+
apiVersion: cluster.x-k8s.io/v1beta1
57+
kind: MachineDeployment
58+
metadata:
59+
annotations:
60+
caution: "# do not use here, because this annotation won't be propagated to your ProxmoxMachine"
61+
name: sample-machine-deployment
62+
spec:
63+
template:
64+
metadata:
65+
annotations:
66+
node.qemu-scheduler/regex: node[0-9]+ # this annotation will be propagated to your ProxmoxMachine via MachineSet
67+
```
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package framework
2+
3+
import (
4+
"github.com/sp-yduck/proxmox-go/api"
5+
"github.com/sp-yduck/proxmox-go/proxmox"
6+
)
7+
8+
type CycleState struct {
9+
completed bool
10+
err error
11+
messages map[string]string
12+
result SchedulerResult
13+
}
14+
15+
type SchedulerResult struct {
16+
vmid int
17+
node string
18+
instance *proxmox.VirtualMachine
19+
}
20+
21+
func NewCycleState() CycleState {
22+
return CycleState{completed: false, err: nil, messages: map[string]string{}}
23+
}
24+
25+
func (c *CycleState) SetComplete() {
26+
c.completed = true
27+
}
28+
29+
func (c *CycleState) IsCompleted() bool {
30+
return c.completed
31+
}
32+
33+
func (c *CycleState) SetError(err error) {
34+
c.err = err
35+
}
36+
37+
func (c *CycleState) Error() error {
38+
return c.err
39+
}
40+
41+
func (c *CycleState) SetMessage(pluginName, message string) {
42+
c.messages[pluginName] = message
43+
}
44+
45+
func (c *CycleState) Messages() map[string]string {
46+
return c.messages
47+
}
48+
49+
func (c *CycleState) QEMU() *api.VirtualMachine {
50+
return c.result.instance.VM
51+
}
52+
53+
func (c *CycleState) UpdateState(completed bool, err error, result SchedulerResult) {
54+
c.completed = completed
55+
c.err = err
56+
c.result = result
57+
}
58+
59+
func NewSchedulerResult(vmid int, node string, instance *proxmox.VirtualMachine) SchedulerResult {
60+
return SchedulerResult{vmid: vmid, node: node, instance: instance}
61+
}
62+
63+
func (c *CycleState) Result() SchedulerResult {
64+
return c.result
65+
}
66+
67+
func (r *SchedulerResult) Node() string {
68+
return r.node
69+
}
70+
71+
func (r *SchedulerResult) VMID() int {
72+
return r.vmid
73+
}
74+
75+
func (r *SchedulerResult) Instance() *proxmox.VirtualMachine {
76+
return r.instance
77+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package framework
2+
3+
import (
4+
"context"
5+
6+
"github.com/sp-yduck/proxmox-go/api"
7+
)
8+
9+
type Plugin interface {
10+
// return plugin name
11+
Name() string
12+
}
13+
14+
type NodeFilterPlugin interface {
15+
Plugin
16+
Filter(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nodeInfo *NodeInfo) *Status
17+
}
18+
19+
type NodeScorePlugin interface {
20+
Plugin
21+
Score(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nodeInfo *NodeInfo) (int64, *Status)
22+
}
23+
24+
type VMIDPlugin interface {
25+
Plugin
26+
PluginKey() CtxKey
27+
Select(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nextid int, usedID map[int]bool) (int, error)
28+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package framework_test
2+
3+
import (
4+
"os"
5+
"testing"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
"github.com/sp-yduck/proxmox-go/proxmox"
10+
logf "sigs.k8s.io/controller-runtime/pkg/log"
11+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
12+
)
13+
14+
var (
15+
proxmoxSvc *proxmox.Service
16+
)
17+
18+
func TestFrameworks(t *testing.T) {
19+
RegisterFailHandler(Fail)
20+
RunSpecs(t, "Scheduler Framework Suite")
21+
}
22+
23+
var _ = BeforeSuite(func() {
24+
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
25+
26+
if GinkgoLabelFilter() != "unit" {
27+
By("setup proxmox client to do integration test")
28+
url := os.Getenv("PROXMOX_URL")
29+
user := os.Getenv("PROXMOX_USER")
30+
password := os.Getenv("PROXMOX_PASSWORD")
31+
tokenid := os.Getenv("PROXMOX_TOKENID")
32+
secret := os.Getenv("PROXMOX_SECRET")
33+
34+
authConfig := proxmox.AuthConfig{
35+
Username: user,
36+
Password: password,
37+
TokenID: tokenid,
38+
Secret: secret,
39+
}
40+
param := proxmox.NewParams(url, authConfig, proxmox.ClientConfig{InsecureSkipVerify: true})
41+
var err error
42+
proxmoxSvc, err = proxmox.GetOrCreateService(param)
43+
Expect(err).NotTo(HaveOccurred())
44+
}
45+
})

cloud/scheduler/framework/types.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package framework
2+
3+
import (
4+
"context"
5+
6+
"github.com/sp-yduck/proxmox-go/api"
7+
"github.com/sp-yduck/proxmox-go/proxmox"
8+
)
9+
10+
type Status struct {
11+
code int
12+
reasons []string
13+
err error
14+
failedPlugin string
15+
}
16+
17+
func NewStatus() *Status {
18+
return &Status{code: 0}
19+
}
20+
21+
func (s *Status) Code() int {
22+
return s.code
23+
}
24+
25+
func (s *Status) SetCode(code int) {
26+
s.code = code
27+
}
28+
29+
func (s *Status) Reasons() []string {
30+
if s.err != nil {
31+
return append([]string{s.err.Error()}, s.reasons...)
32+
}
33+
return s.reasons
34+
}
35+
36+
func (s *Status) FailedPlugin() string {
37+
return s.failedPlugin
38+
}
39+
40+
func (s *Status) SetFailedPlugin(name string) {
41+
s.failedPlugin = name
42+
}
43+
44+
func (s *Status) IsSuccess() bool {
45+
return s.code == 0
46+
}
47+
48+
func (s *Status) Error() error {
49+
return s.err
50+
}
51+
52+
// NodeInfo is node level aggregated information
53+
type NodeInfo struct {
54+
node *api.Node
55+
56+
// qemus assigned to the node
57+
qemus []*api.VirtualMachine
58+
}
59+
60+
func GetNodeInfoList(ctx context.Context, client *proxmox.Service) ([]*NodeInfo, error) {
61+
nodes, err := client.Nodes(ctx)
62+
if err != nil {
63+
return nil, err
64+
}
65+
nodeInfos := []*NodeInfo{}
66+
for _, node := range nodes {
67+
qemus, err := client.RESTClient().GetVirtualMachines(ctx, node.Node)
68+
if err != nil {
69+
return nil, err
70+
}
71+
nodeInfos = append(nodeInfos, &NodeInfo{node: node, qemus: qemus})
72+
}
73+
return nodeInfos, nil
74+
}
75+
76+
func (n NodeInfo) Node() *api.Node {
77+
return n.node
78+
}
79+
80+
func (n NodeInfo) QEMUs() []*api.VirtualMachine {
81+
return n.qemus
82+
}
83+
84+
// NodeScoreList declares a list of nodes and their scores.
85+
type NodeScoreList []NodeScore
86+
87+
// NodeScore is a struct with node name and score.
88+
type NodeScore struct {
89+
Name string
90+
Score int64
91+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package framework_test
2+
3+
import (
4+
"context"
5+
6+
. "github.com/onsi/ginkgo/v2"
7+
. "github.com/onsi/gomega"
8+
9+
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
10+
)
11+
12+
var _ = Describe("GetNodeInfoList", Label("integration", "framework"), func() {
13+
ctx := context.Background()
14+
15+
It("should not error", func() {
16+
nodes, err := framework.GetNodeInfoList(ctx, proxmoxSvc)
17+
Expect(err).To(BeNil())
18+
Expect(len(nodes)).ToNot(Equal(0))
19+
})
20+
21+
})

0 commit comments

Comments
 (0)