Skip to content

Commit 36be6cf

Browse files
reshnmreshnm
authored
add crd management library (#47)
1 parent 7f481a2 commit 36be6cf

File tree

5 files changed

+318
-1
lines changed

5 files changed

+318
-1
lines changed

‎docs/libs/crds.md‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,62 @@
11
# Custom Resource Definitions
22

3-
The `pkg/init/crds` package allows user to deploy CRDs from yaml files to a target cluster. It uses `embed.FS` to provide the files for deployment.
3+
The `pkg/crds` package allows user to deploy CRDs from yaml files to a target cluster.
4+
A typical use case is to use `embed.FS` to embed the CRDs in the controller binary and deploy them to the target clusters.
5+
6+
The decision on which cluster a CRD should be deployed to is made by the `CRDManager` based on the labels of the CRDs and the labels of the clusters.
7+
The label key is passed to the `CRDManager` when it is created.
8+
Each cluster is then registered with a label value at the `CRDManager`.
9+
10+
## Example
11+
12+
```go
13+
package main
14+
15+
import (
16+
"context"
17+
"embed"
18+
19+
"github.com/openmcp-project/controller-utils/pkg/clusters"
20+
"github.com/openmcp-project/controller-utils/pkg/crds"
21+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
22+
)
23+
24+
//go:embed crds
25+
var crdsFS embed.FS
26+
var crdsPath = "crds"
27+
28+
func main() {
29+
ctx := context.Background()
30+
31+
onboardingCluster := clusters.NewTestClusterFromClient("onboarding", getOnboardingClient())
32+
workloadCluster := clusters.NewTestClusterFromClient("workload", getWorkloadClient())
33+
34+
// use "openmcp.cloud/cluster" as the CRD label key
35+
crdManager := crds.NewCRDManager("openmcp.cloud/cluster", func() ([]*apiextv1.CustomResourceDefinition, error) {
36+
return crds.CRDsFromFileSystem(crdsFS, crdsPath)
37+
})
38+
39+
// register the onboarding cluster with label value "onboarding"
40+
crdManager.AddCRDLabelToClusterMapping("onboarding", onboardingCluster)
41+
// register the workload cluster with label value "workload"
42+
crdManager.AddCRDLabelToClusterMapping("workload", workloadCluster)
43+
44+
// create/update the CRDs in all clusters
45+
err := crdManager.CreateOrUpdateCRDs(ctx, nil)
46+
if err != nil {
47+
panic(err)
48+
}
49+
}
50+
```
51+
52+
The CRDs need to be annotated with the label key and the label value of the cluster they should be deployed to.
53+
54+
```yaml
55+
apiVersion: apiextensions.k8s.io/v1
56+
kind: CustomResourceDefinition
57+
metadata:
58+
name: testresources.example.com
59+
labels:
60+
openmcp.cloud/cluster: "onboarding"
61+
...
62+
```

‎pkg/crds/crds.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package crds
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io/fs"
7+
"path/filepath"
8+
9+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
10+
"sigs.k8s.io/yaml"
11+
12+
"github.com/openmcp-project/controller-utils/pkg/clusters"
13+
"github.com/openmcp-project/controller-utils/pkg/controller"
14+
"github.com/openmcp-project/controller-utils/pkg/errors"
15+
"github.com/openmcp-project/controller-utils/pkg/logging"
16+
"github.com/openmcp-project/controller-utils/pkg/resources"
17+
)
18+
19+
type (
20+
CRDLabelToClusterMappings map[string]*clusters.Cluster
21+
CRDList func() ([]*apiextv1.CustomResourceDefinition, error)
22+
)
23+
24+
type CRDManager struct {
25+
mappingLabelName string
26+
crdLabelsToClusterMappings CRDLabelToClusterMappings
27+
crdList CRDList
28+
}
29+
30+
func NewCRDManager(mappingLabelName string, crdList CRDList) *CRDManager {
31+
return &CRDManager{
32+
mappingLabelName: mappingLabelName,
33+
crdLabelsToClusterMappings: make(CRDLabelToClusterMappings),
34+
crdList: crdList,
35+
}
36+
}
37+
38+
func (m *CRDManager) AddCRDLabelToClusterMapping(labelValue string, cluster *clusters.Cluster) {
39+
m.crdLabelsToClusterMappings[labelValue] = cluster
40+
}
41+
42+
func (m *CRDManager) CreateOrUpdateCRDs(ctx context.Context, log *logging.Logger) error {
43+
crds, err := m.crdList()
44+
if err != nil {
45+
return fmt.Errorf("error getting CRDs: %w", err)
46+
}
47+
48+
var errs error
49+
50+
for _, crd := range crds {
51+
c, err := m.getClusterForCRD(crd)
52+
if err != nil {
53+
errs = errors.Join(errs, err)
54+
continue
55+
}
56+
57+
if log != nil {
58+
log.Info("creating/updating CRD", "name", crd.Name, "cluster", c.ID())
59+
}
60+
err = resources.CreateOrUpdateResource(ctx, c.Client(), resources.NewCRDMutator(crd, crd.Labels, crd.Annotations))
61+
errs = errors.Join(errs, err)
62+
}
63+
64+
if errs != nil {
65+
return fmt.Errorf("error creating/updating CRDs: %w", errs)
66+
}
67+
return nil
68+
}
69+
70+
func (m *CRDManager) getClusterForCRD(crd *apiextv1.CustomResourceDefinition) (*clusters.Cluster, error) {
71+
labelValue, ok := controller.GetLabel(crd, m.mappingLabelName)
72+
if !ok {
73+
return nil, fmt.Errorf("missing label '%s' for CRD '%s'", m.mappingLabelName, crd.Name)
74+
}
75+
76+
cluster, ok := m.crdLabelsToClusterMappings[labelValue]
77+
if !ok {
78+
return nil, fmt.Errorf("no cluster mapping found for label value '%s' in CRD '%s'", labelValue, crd.Name)
79+
}
80+
81+
return cluster, nil
82+
}
83+
84+
// CRDsFromFileSystem reads CRDs from the specified filesystem path.
85+
func CRDsFromFileSystem(fsys fs.FS, path string) ([]*apiextv1.CustomResourceDefinition, error) {
86+
var crds []*apiextv1.CustomResourceDefinition
87+
88+
entries, err := fs.ReadDir(fsys, path)
89+
if err != nil {
90+
return nil, fmt.Errorf("failed to read directory %s: %w", path, err)
91+
}
92+
93+
for _, entry := range entries {
94+
if entry.IsDir() {
95+
continue
96+
}
97+
98+
filePath := filepath.Join(path, entry.Name())
99+
data, err := fs.ReadFile(fsys, filePath)
100+
if err != nil {
101+
return nil, fmt.Errorf("failed to read file %s: %w", filePath, err)
102+
}
103+
104+
var crd apiextv1.CustomResourceDefinition
105+
if err := yaml.Unmarshal(data, &crd); err != nil {
106+
return nil, fmt.Errorf("failed to unmarshal CRD from file %s: %w", filePath, err)
107+
}
108+
109+
crds = append(crds, &crd)
110+
}
111+
112+
return crds, nil
113+
}

‎pkg/crds/crds_test.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package crds_test
2+
3+
import (
4+
"context"
5+
"embed"
6+
"testing"
7+
8+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
9+
"k8s.io/apimachinery/pkg/runtime"
10+
"k8s.io/apimachinery/pkg/types"
11+
12+
"github.com/openmcp-project/controller-utils/pkg/clusters"
13+
utilstest "github.com/openmcp-project/controller-utils/pkg/testing"
14+
15+
. "github.com/onsi/ginkgo/v2"
16+
. "github.com/onsi/gomega"
17+
18+
"github.com/openmcp-project/controller-utils/pkg/crds"
19+
)
20+
21+
//go:embed testdata/*
22+
var testFS embed.FS
23+
24+
func TestCRDs(t *testing.T) {
25+
RegisterFailHandler(Fail)
26+
RunSpecs(t, "CRDs Test Suite")
27+
}
28+
29+
var _ = Describe("CRDsFromFileSystem", func() {
30+
It("should correctly read and parse CRDs from the filesystem", func() {
31+
crdPath := "testdata"
32+
crdsList, err := crds.CRDsFromFileSystem(testFS, crdPath)
33+
Expect(err).NotTo(HaveOccurred())
34+
Expect(crdsList).To(HaveLen(2))
35+
36+
// Validate the first CRD
37+
Expect(crdsList[0].Name).To(Equal("testresources.example.com"))
38+
Expect(crdsList[0].Spec.Names.Kind).To(Equal("TestResource"))
39+
40+
// Validate the second CRD
41+
Expect(crdsList[1].Name).To(Equal("sampleresources.example.com"))
42+
Expect(crdsList[1].Spec.Names.Kind).To(Equal("SampleResource"))
43+
})
44+
})
45+
46+
var _ = Describe("CRDManager", func() {
47+
It("should correctly manage CRD mappings and create/update CRDs", func() {
48+
scheme := runtime.NewScheme()
49+
err := apiextv1.AddToScheme(scheme)
50+
Expect(err).NotTo(HaveOccurred())
51+
52+
// Create fake clients
53+
clientA, err := utilstest.GetFakeClient(scheme)
54+
Expect(err).NotTo(HaveOccurred())
55+
56+
clientB, err := utilstest.GetFakeClient(scheme)
57+
Expect(err).NotTo(HaveOccurred())
58+
59+
// Create fake clusters
60+
clusterA := clusters.NewTestClusterFromClient("cluster_a", clientA)
61+
clusterB := clusters.NewTestClusterFromClient("cluster_b", clientB)
62+
63+
crdManager := crds.NewCRDManager("openmcp.cloud/cluster", func() ([]*apiextv1.CustomResourceDefinition, error) {
64+
return crds.CRDsFromFileSystem(testFS, "testdata")
65+
})
66+
67+
crdManager.AddCRDLabelToClusterMapping("cluster_a", clusterA)
68+
crdManager.AddCRDLabelToClusterMapping("cluster_b", clusterB)
69+
70+
ctx := context.Background()
71+
72+
err = crdManager.CreateOrUpdateCRDs(ctx, nil)
73+
Expect(err).NotTo(HaveOccurred())
74+
75+
// Verify that the CRDs were created in the respective clusters
76+
crdA := &apiextv1.CustomResourceDefinition{}
77+
err = clientA.Get(ctx, types.NamespacedName{Name: "testresources.example.com"}, crdA)
78+
Expect(err).NotTo(HaveOccurred())
79+
Expect(crdA.Labels["openmcp.cloud/cluster"]).To(Equal("cluster_a"))
80+
81+
crdB := &apiextv1.CustomResourceDefinition{}
82+
err = clientB.Get(ctx, types.NamespacedName{Name: "sampleresources.example.com"}, crdB)
83+
Expect(err).NotTo(HaveOccurred())
84+
Expect(crdB.Labels["openmcp.cloud/cluster"]).To(Equal("cluster_b"))
85+
})
86+
})

‎pkg/crds/testdata/crd_a.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: apiextensions.k8s.io/v1
2+
kind: CustomResourceDefinition
3+
metadata:
4+
name: testresources.example.com
5+
labels:
6+
openmcp.cloud/cluster: "cluster_a"
7+
spec:
8+
group: example.com
9+
names:
10+
kind: TestResource
11+
listKind: TestResourceList
12+
plural: testresources
13+
singular: testresource
14+
scope: Namespaced
15+
versions:
16+
- name: v1
17+
served: true
18+
storage: true
19+
schema:
20+
openAPIV3Schema:
21+
type: object
22+
properties:
23+
spec:
24+
type: object
25+
properties:
26+
name:
27+
type: string
28+
replicas:
29+
type: integer
30+
minimum: 1

‎pkg/crds/testdata/crd_b.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
apiVersion: apiextensions.k8s.io/v1
2+
kind: CustomResourceDefinition
3+
metadata:
4+
name: sampleresources.example.com
5+
labels:
6+
openmcp.cloud/cluster: "cluster_b"
7+
spec:
8+
group: example.com
9+
names:
10+
kind: SampleResource
11+
listKind: SampleResourceList
12+
plural: sampleresources
13+
singular: sampleresource
14+
scope: Namespaced
15+
versions:
16+
- name: v1
17+
served: true
18+
storage: true
19+
schema:
20+
openAPIV3Schema:
21+
type: object
22+
properties:
23+
spec:
24+
type: object
25+
properties:
26+
type:
27+
type: string
28+
enabled:
29+
type: boolean

0 commit comments

Comments
 (0)