fix scheduler logics

Author: sp-yduck

README.md

@@ -63,6 +63,8 @@ kubectl delete cluster cappx-test
 
 - Supports custom cloud-config (user data). CAPPX uses VNC websockert for bootstrapping nodes so it can applies custom cloud-config that can not be achieved by only Proxmox API.
 
+- Flexible vmid/node assigning. You can flexibly assign vmid to your qemu and flexibly schedule qemus to proxmox nodes. For more details please check [qemu-scheduler](./cloud/scheduler/).
+
 ### Node Images
 
 CAPPX is compatible with `iso`, `qcow2`, `qed`, `raw`, `vdi`, `vpc`, `vmdk` format of image. You can build your own node image and use it for `ProxmoxMachine`.

cloud/scheduler/README.md

@@ -2,18 +2,38 @@
 
 Scheduling refers to making sure that VM(QEMU) are matched to Proxmox Nodes.
 
+## How qemu-scheduler select proxmox node to run qemu
+
+Basic flow of the node selection process is `filter => score => select one node which has highest score`
+
+### Filter Plugins
+
+Filter plugins filter the node based on nodename, overcommit ratio etc.
+
+#### regex plugin
+
+Regex plugin is a one of the default Filter Plugin of qemu-scheduler. You can specify node name as regex format. 
+```sh
+key: node.qemu-scheduler/reges
+value(example): node[0-9]+
+```
+
+### Score Plugins
+
+Score plugins score the nodes based on resource etc.
+
 ## How to specify vmid
 qemu-scheduler reads context and find key registerd to scheduler. If the context has any value of the registerd key, qemu-scheduler uses the plugin that matchies the key.
 
 ### Range Plugin
 You can specify vmid range with `(start id)-(end id)` format.
-```
+```sh
 key: vmid.qemu-scheduler/range
 value(example): 100-150
 ```
 
 ### Regex Plugin
-```
+```sh
 key: vmid.qemu-scheduler/regex
 value(example): (12[0-9]|130)
 ```
@@ -43,5 +63,5 @@ spec:
   template:
     metadata:
       annotations:
-        vmid.qemu-scheduler/range: 100-150 # this annotation will be propagated to your ProxmoxMachine via MachineSet
+        node.qemu-scheduler/regex: node[0-9]+ # this annotation will be propagated to your ProxmoxMachine via MachineSet
 ```

cloud/scheduler/framework/cycle_state.go

@@ -41,10 +41,10 @@ func (c *CycleState) QEMU() *api.VirtualMachine {
 	return c.result.instance.VM
 }
 
-func (c *CycleState) UpdateState(completed bool, err error, result *SchedulerResult) {
+func (c *CycleState) UpdateState(completed bool, err error, result SchedulerResult) {
 	c.completed = completed
 	c.err = err
-	c.result = *result
+	c.result = result
 }
 
 func NewSchedulerResult(vmid int, node string, instance *proxmox.VirtualMachine) SchedulerResult {

cloud/scheduler/framework/types.go

@@ -77,6 +77,10 @@ func (n NodeInfo) Node() *api.Node {
 	return n.node
 }
 
+func (n NodeInfo) QEMUs() []*api.VirtualMachine {
+	return n.qemus
+}
+
 // NodeScoreList declares a list of nodes and their scores.
 type NodeScoreList []NodeScore
 

cloud/scheduler/plugins/names/names.go

@@ -3,15 +3,24 @@ package names
 // node plugins
 const (
 	// filter plugins
-
+	// filter by node name
 	NodeName = "NodeName"
+	// filter by node name regex
+	NodeRegex = "NodeRegex"
+	// filter by cpu overcommit ratio
+	CPUOvercommit = "CPUOvercommit"
+	// filter by memory overcommit ratio
+	MemoryOvercommit = "MemoryOvercommit"
 
 	// score plugins
-
+	// random score
 	Random = "Random"
+	// resource utilization score
+	NodeResource = "NodeResource"
 
 	// vmid plugins
-
+	// select by range
 	Range = "Range"
+	// select by regex
 	Regex = "Regex"
 )

cloud/scheduler/plugins/noderesource/node_resrouce.go

@@ -0,0 +1,32 @@
+package noderesource
+
+import (
+	"context"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
+	"github.com/sp-yduck/proxmox-go/api"
+)
+
+type NodeResource struct{}
+
+var _ framework.NodeScorePlugin = &NodeResource{}
+
+const (
+	Name = names.NodeResource
+)
+
+func (pl *NodeResource) Name() string {
+	return Name
+}
+
+// score = 1/(cpu/maxcpu * mem/maxmem)
+func (pl *NodeResource) Score(ctx context.Context, state *framework.CycleState, _ api.VirtualMachineCreateOptions, nodeInfo *framework.NodeInfo) (int64, *framework.Status) {
+	cpu := nodeInfo.Node().Cpu
+	maxCPU := nodeInfo.Node().MaxCpu
+	mem := nodeInfo.Node().Mem
+	maxMem := nodeInfo.Node().MaxMem
+	u := cpu / float32(maxCPU) * float32(mem/maxMem)
+	score := int64(1 / u)
+	return score, nil
+}

cloud/scheduler/plugins/overcommit/cpu_overcommit.go

@@ -0,0 +1,48 @@
+package overcommit
+
+import (
+	"context"
+
+	"github.com/sp-yduck/proxmox-go/api"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
+)
+
+type CPUOvercommit struct{}
+
+var _ framework.NodeFilterPlugin = &CPUOvercommit{}
+
+const (
+	CPUOvercommitName         = names.CPUOvercommit
+	defaultCPUOvercommitRatio = 4
+)
+
+func (pl *CPUOvercommit) Name() string {
+	return CPUOvercommitName
+}
+
+// filter by cpu overcommit ratio
+func (pl *CPUOvercommit) Filter(ctx context.Context, _ *framework.CycleState, config api.VirtualMachineCreateOptions, nodeInfo *framework.NodeInfo) *framework.Status {
+	cpu := sumCPUs(nodeInfo.QEMUs())
+	maxCPU := nodeInfo.Node().MaxCpu
+	sockets := config.Sockets
+	if sockets == 0 {
+		sockets = 1
+	}
+	ratio := float32(cpu+config.Cores*sockets) / float32(maxCPU)
+	if ratio > defaultCPUOvercommitRatio {
+		status := framework.NewStatus()
+		status.SetCode(1)
+		return status
+	}
+	return &framework.Status{}
+}
+
+func sumCPUs(qemus []*api.VirtualMachine) int {
+	var result int
+	for _, q := range qemus {
+		result += q.Cpus
+	}
+	return result
+}

cloud/scheduler/plugins/overcommit/memory_overcommit.go

@@ -0,0 +1,44 @@
+package overcommit
+
+import (
+	"context"
+
+	"github.com/sp-yduck/proxmox-go/api"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
+)
+
+type MemoryOvercommit struct{}
+
+var _ framework.NodeFilterPlugin = &MemoryOvercommit{}
+
+const (
+	MemoryOvercommitName         = names.MemoryOvercommit
+	defaultMemoryOvercommitRatio = 1
+)
+
+func (pl *MemoryOvercommit) Name() string {
+	return MemoryOvercommitName
+}
+
+// filter by memory overcommit ratio
+func (pl *MemoryOvercommit) Filter(ctx context.Context, _ *framework.CycleState, config api.VirtualMachineCreateOptions, nodeInfo *framework.NodeInfo) *framework.Status {
+	mem := sumMems(nodeInfo.QEMUs())
+	maxMem := nodeInfo.Node().MaxMem
+	ratio := float32(mem+1024*1024*config.Memory) / float32(maxMem)
+	if ratio >= defaultMemoryOvercommitRatio {
+		status := framework.NewStatus()
+		status.SetCode(1)
+		return status
+	}
+	return &framework.Status{}
+}
+
+func sumMems(qemus []*api.VirtualMachine) int {
+	var result int
+	for _, q := range qemus {
+		result += q.MaxMem
+	}
+	return result
+}

cloud/scheduler/plugins/regex/node_regex.go

@@ -0,0 +1,54 @@
+package regex
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/sp-yduck/proxmox-go/api"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/names"
+)
+
+type NodeRegex struct{}
+
+var _ framework.NodeFilterPlugin = &NodeRegex{}
+
+const (
+	NodeRegexName = names.NodeRegex
+	NodeRegexKey  = "node.qemu-scheduler/regex"
+)
+
+func (pl *NodeRegex) Name() string {
+	return NodeRegexName
+}
+
+// regex is specified in ctx value (key=node.qemu-scheduler/regex)
+func (pl *NodeRegex) Filter(ctx context.Context, _ *framework.CycleState, config api.VirtualMachineCreateOptions, nodeInfo *framework.NodeInfo) *framework.Status {
+	reg, err := findNodeRegex(ctx)
+	if err != nil {
+
+		return &framework.Status{}
+	}
+	if !reg.MatchString(nodeInfo.Node().Node) {
+		status := framework.NewStatus()
+		status.SetCode(1)
+		return status
+	}
+	return &framework.Status{}
+}
+
+// specify available node name as regex
+// example: node.qemu-scheduler/regex=node[0-9]+
+func findNodeRegex(ctx context.Context) (*regexp.Regexp, error) {
+	value := ctx.Value(framework.CtxKey(NodeRegexKey))
+	if value == nil {
+		return nil, fmt.Errorf("no node name regex is specified")
+	}
+	reg, err := regexp.Compile(fmt.Sprintf("%s", value))
+	if err != nil {
+		return nil, err
+	}
+	return reg, nil
+}