Skip to content
This repository was archived by the owner on Oct 14, 2020. It is now read-only.

Commit d077709

Browse files
nigthknightnigthknight
committed
Split scan_controller
1 parent 7157ae2 commit d077709

File tree

2 files changed

+191
-173
lines changed

2 files changed

+191
-173
lines changed

‎operator/controllers/execution/scans/parse_reconciler.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
package scancontrollers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
executionv1 "github.com/secureCodeBox/secureCodeBox-v2-alpha/operator/apis/execution/v1"
9+
util "github.com/secureCodeBox/secureCodeBox-v2-alpha/operator/utils"
10+
batch "k8s.io/api/batch/v1"
11+
corev1 "k8s.io/api/core/v1"
12+
rbacv1 "k8s.io/api/rbac/v1"
13+
resource "k8s.io/apimachinery/pkg/api/resource"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/types"
16+
ctrl "sigs.k8s.io/controller-runtime"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
18+
)
19+
20+
func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
21+
ctx := context.Background()
22+
namespacedName := fmt.Sprintf("%s/%s", scan.Namespace, scan.Name)
23+
log := r.Log.WithValues("scan_parse", namespacedName)
24+
25+
jobs, err := r.getJobsForScan(scan, client.MatchingLabels{"experimental.securecodebox.io/job-type": "parser"})
26+
if err != nil {
27+
return err
28+
}
29+
if len(jobs.Items) > 0 {
30+
log.V(8).Info("Job already exists. Doesn't need to be created.")
31+
return nil
32+
}
33+
34+
parseType := scan.Status.RawResultType
35+
36+
// get the scan template for the scan
37+
var parseDefinition executionv1.ParseDefinition
38+
if err := r.Get(ctx, types.NamespacedName{Name: parseType, Namespace: scan.Namespace}, &parseDefinition); err != nil {
39+
log.V(7).Info("Unable to fetch ParseDefinition")
40+
41+
scan.Status.State = "Errored"
42+
scan.Status.ErrorDescription = fmt.Sprintf("No ParseDefinition for ResultType '%s' found in Scans Namespace.", parseType)
43+
if err := r.Status().Update(ctx, scan); err != nil {
44+
r.Log.Error(err, "unable to update Scan status")
45+
return err
46+
}
47+
48+
return fmt.Errorf("No ParseDefinition of type '%s' found", parseType)
49+
}
50+
log.Info("Matching ParseDefinition Found", "ParseDefinition", parseType)
51+
52+
findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json")
53+
if err != nil {
54+
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
55+
return err
56+
}
57+
rawResultDownloadURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile)
58+
if err != nil {
59+
return err
60+
}
61+
62+
rules := []rbacv1.PolicyRule{
63+
{
64+
APIGroups: []string{"execution.experimental.securecodebox.io"},
65+
Resources: []string{"scans/status"},
66+
Verbs: []string{"get", "patch"},
67+
},
68+
}
69+
r.ensureServiceAccountExists(
70+
scan.Namespace,
71+
"parser",
72+
"Parser need to access the status of Scans to update how many findings have been identified",
73+
rules,
74+
)
75+
76+
labels := scan.ObjectMeta.DeepCopy().Labels
77+
if labels == nil {
78+
labels = make(map[string]string)
79+
}
80+
labels["experimental.securecodebox.io/job-type"] = "parser"
81+
automountServiceAccountToken := true
82+
var backOffLimit int32 = 3
83+
job := &batch.Job{
84+
ObjectMeta: metav1.ObjectMeta{
85+
Annotations: make(map[string]string),
86+
GenerateName: util.TruncateName(fmt.Sprintf("parse-%s", scan.Name)),
87+
Namespace: scan.Namespace,
88+
Labels: labels,
89+
},
90+
Spec: batch.JobSpec{
91+
BackoffLimit: &backOffLimit,
92+
Template: corev1.PodTemplateSpec{
93+
ObjectMeta: metav1.ObjectMeta{
94+
Annotations: map[string]string{
95+
"auto-discovery.experimental.securecodebox.io/ignore": "true",
96+
},
97+
},
98+
Spec: corev1.PodSpec{
99+
RestartPolicy: corev1.RestartPolicyNever,
100+
ServiceAccountName: "parser",
101+
ImagePullSecrets: parseDefinition.Spec.ImagePullSecrets,
102+
Containers: []corev1.Container{
103+
{
104+
Name: "parser",
105+
Image: parseDefinition.Spec.Image,
106+
Env: []corev1.EnvVar{
107+
{
108+
Name: "NAMESPACE",
109+
ValueFrom: &corev1.EnvVarSource{
110+
FieldRef: &corev1.ObjectFieldSelector{
111+
FieldPath: "metadata.namespace",
112+
},
113+
},
114+
},
115+
{
116+
Name: "SCAN_NAME",
117+
Value: scan.Name,
118+
},
119+
},
120+
Args: []string{
121+
rawResultDownloadURL,
122+
findingsUploadURL,
123+
},
124+
ImagePullPolicy: "Always",
125+
Resources: corev1.ResourceRequirements{
126+
Requests: corev1.ResourceList{
127+
corev1.ResourceCPU: resource.MustParse("200m"),
128+
corev1.ResourceMemory: resource.MustParse("100Mi"),
129+
},
130+
Limits: corev1.ResourceList{
131+
corev1.ResourceCPU: resource.MustParse("400m"),
132+
corev1.ResourceMemory: resource.MustParse("200Mi"),
133+
},
134+
},
135+
},
136+
},
137+
AutomountServiceAccountToken: &automountServiceAccountToken,
138+
},
139+
},
140+
TTLSecondsAfterFinished: nil,
141+
},
142+
}
143+
144+
if err := ctrl.SetControllerReference(scan, job, r.Scheme); err != nil {
145+
return err
146+
}
147+
148+
log.V(7).Info("Constructed Job object", "job args", strings.Join(job.Spec.Template.Spec.Containers[0].Args, ", "))
149+
150+
if err := r.Create(ctx, job); err != nil {
151+
log.Error(err, "unable to create Job for Parser", "job", job)
152+
return err
153+
}
154+
155+
scan.Status.State = "Parsing"
156+
if err := r.Status().Update(ctx, scan); err != nil {
157+
log.Error(err, "unable to update Scan status")
158+
return err
159+
}
160+
161+
log.V(1).Info("created Parse Job for Scan", "job", job)
162+
return nil
163+
}
164+
165+
func (r *ScanReconciler) checkIfParsingIsCompleted(scan *executionv1.Scan) error {
166+
ctx := context.Background()
167+
168+
status, err := r.checkIfJobIsCompleted(scan, client.MatchingLabels{"experimental.securecodebox.io/job-type": "parser"})
169+
if err != nil {
170+
return err
171+
}
172+
173+
switch status {
174+
case completed:
175+
r.Log.V(7).Info("Parsing is completed")
176+
scan.Status.State = "ParseCompleted"
177+
if err := r.Status().Update(ctx, scan); err != nil {
178+
r.Log.Error(err, "unable to update Scan status")
179+
return err
180+
}
181+
case failed:
182+
scan.Status.State = "Errored"
183+
scan.Status.ErrorDescription = "Failed to run the Parser. This is likely a Bug, we would like to know about. Please open up a Issue on GitHub."
184+
if err := r.Status().Update(ctx, scan); err != nil {
185+
r.Log.Error(err, "unable to update Scan status")
186+
return err
187+
}
188+
}
189+
190+
return nil
191+
}

‎operator/controllers/execution/scans/scan_controller.go‎

Lines changed: 0 additions & 173 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"net/url"
2424
"os"
2525
"path/filepath"
26-
"strings"
2726
"time"
2827

2928
"github.com/go-logr/logr"
@@ -210,179 +209,7 @@ func (r *ScanReconciler) handleFinalizer(scan *executionv1.Scan) error {
210209
return nil
211210
}
212211

213-
func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
214-
ctx := context.Background()
215-
namespacedName := fmt.Sprintf("%s/%s", scan.Namespace, scan.Name)
216-
log := r.Log.WithValues("scan_parse", namespacedName)
217-
218-
jobs, err := r.getJobsForScan(scan, client.MatchingLabels{"experimental.securecodebox.io/job-type": "parser"})
219-
if err != nil {
220-
return err
221-
}
222-
if len(jobs.Items) > 0 {
223-
log.V(8).Info("Job already exists. Doesn't need to be created.")
224-
return nil
225-
}
226-
227-
parseType := scan.Status.RawResultType
228-
229-
// get the scan template for the scan
230-
var parseDefinition executionv1.ParseDefinition
231-
if err := r.Get(ctx, types.NamespacedName{Name: parseType, Namespace: scan.Namespace}, &parseDefinition); err != nil {
232-
log.V(7).Info("Unable to fetch ParseDefinition")
233-
234-
scan.Status.State = "Errored"
235-
scan.Status.ErrorDescription = fmt.Sprintf("No ParseDefinition for ResultType '%s' found in Scans Namespace.", parseType)
236-
if err := r.Status().Update(ctx, scan); err != nil {
237-
r.Log.Error(err, "unable to update Scan status")
238-
return err
239-
}
240-
241-
return fmt.Errorf("No ParseDefinition of type '%s' found", parseType)
242-
}
243-
log.Info("Matching ParseDefinition Found", "ParseDefinition", parseType)
244-
245-
findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json")
246-
if err != nil {
247-
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
248-
return err
249-
}
250-
rawResultDownloadURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile)
251-
if err != nil {
252-
return err
253-
}
254-
255-
rules := []rbacv1.PolicyRule{
256-
{
257-
APIGroups: []string{"execution.experimental.securecodebox.io"},
258-
Resources: []string{"scans/status"},
259-
Verbs: []string{"get", "patch"},
260-
},
261-
}
262-
r.ensureServiceAccountExists(
263-
scan.Namespace,
264-
"parser",
265-
"Parser need to access the status of Scans to update how many findings have been identified",
266-
rules,
267-
)
268-
269-
labels := scan.ObjectMeta.DeepCopy().Labels
270-
if labels == nil {
271-
labels = make(map[string]string)
272-
}
273-
labels["experimental.securecodebox.io/job-type"] = "parser"
274-
automountServiceAccountToken := true
275-
var backOffLimit int32 = 3
276-
job := &batch.Job{
277-
ObjectMeta: metav1.ObjectMeta{
278-
Annotations: make(map[string]string),
279-
GenerateName: util.TruncateName(fmt.Sprintf("parse-%s", scan.Name)),
280-
Namespace: scan.Namespace,
281-
Labels: labels,
282-
},
283-
Spec: batch.JobSpec{
284-
BackoffLimit: &backOffLimit,
285-
Template: corev1.PodTemplateSpec{
286-
ObjectMeta: metav1.ObjectMeta{
287-
Annotations: map[string]string{
288-
"auto-discovery.experimental.securecodebox.io/ignore": "true",
289-
},
290-
},
291-
Spec: corev1.PodSpec{
292-
RestartPolicy: corev1.RestartPolicyNever,
293-
ServiceAccountName: "parser",
294-
ImagePullSecrets: parseDefinition.Spec.ImagePullSecrets,
295-
Containers: []corev1.Container{
296-
{
297-
Name: "parser",
298-
Image: parseDefinition.Spec.Image,
299-
Env: []corev1.EnvVar{
300-
{
301-
Name: "NAMESPACE",
302-
ValueFrom: &corev1.EnvVarSource{
303-
FieldRef: &corev1.ObjectFieldSelector{
304-
FieldPath: "metadata.namespace",
305-
},
306-
},
307-
},
308-
{
309-
Name: "SCAN_NAME",
310-
Value: scan.Name,
311-
},
312-
},
313-
Args: []string{
314-
rawResultDownloadURL,
315-
findingsUploadURL,
316-
},
317-
ImagePullPolicy: "Always",
318-
Resources: corev1.ResourceRequirements{
319-
Requests: corev1.ResourceList{
320-
corev1.ResourceCPU: resource.MustParse("200m"),
321-
corev1.ResourceMemory: resource.MustParse("100Mi"),
322-
},
323-
Limits: corev1.ResourceList{
324-
corev1.ResourceCPU: resource.MustParse("400m"),
325-
corev1.ResourceMemory: resource.MustParse("200Mi"),
326-
},
327-
},
328-
},
329-
},
330-
AutomountServiceAccountToken: &automountServiceAccountToken,
331-
},
332-
},
333-
TTLSecondsAfterFinished: nil,
334-
},
335-
}
336-
337-
if err := ctrl.SetControllerReference(scan, job, r.Scheme); err != nil {
338-
return err
339-
}
340-
341-
log.V(7).Info("Constructed Job object", "job args", strings.Join(job.Spec.Template.Spec.Containers[0].Args, ", "))
342-
343-
if err := r.Create(ctx, job); err != nil {
344-
log.Error(err, "unable to create Job for Parser", "job", job)
345-
return err
346-
}
347-
348-
scan.Status.State = "Parsing"
349-
if err := r.Status().Update(ctx, scan); err != nil {
350-
log.Error(err, "unable to update Scan status")
351-
return err
352-
}
353-
354-
log.V(1).Info("created Parse Job for Scan", "job", job)
355-
return nil
356-
}
357-
358212
// Checking if Parser has completed
359-
func (r *ScanReconciler) checkIfParsingIsCompleted(scan *executionv1.Scan) error {
360-
ctx := context.Background()
361-
362-
status, err := r.checkIfJobIsCompleted(scan, client.MatchingLabels{"experimental.securecodebox.io/job-type": "parser"})
363-
if err != nil {
364-
return err
365-
}
366-
367-
switch status {
368-
case completed:
369-
r.Log.V(7).Info("Parsing is completed")
370-
scan.Status.State = "ParseCompleted"
371-
if err := r.Status().Update(ctx, scan); err != nil {
372-
r.Log.Error(err, "unable to update Scan status")
373-
return err
374-
}
375-
case failed:
376-
scan.Status.State = "Errored"
377-
scan.Status.ErrorDescription = "Failed to run the Parser. This is likely a Bug, we would like to know about. Please open up a Issue on GitHub."
378-
if err := r.Status().Update(ctx, scan); err != nil {
379-
r.Log.Error(err, "unable to update Scan status")
380-
return err
381-
}
382-
}
383-
384-
return nil
385-
}
386213

387214
func (r *ScanReconciler) constructJobForScan(scan *executionv1.Scan, scanType *executionv1.ScanType) (*batch.Job, error) {
388215
filename := filepath.Base(scanType.Spec.ExtractResults.Location)

0 commit comments

Comments
 (0)