VEN-8 | Add flags for different crawling options, add new cli commands (#967)

* VEN-8 | Add flags for different crawling options, add new cli commands

Author: bokimilinkovic

test-crawler/Makefile

@@ -3,4 +3,13 @@ run:
 	go run main.go config.go
 
 copy-output:
-	go run main.go config.go > ../frontend/apps/web/src/tests.json
+	go run main.go config.go > ../frontend/apps/web/src/tests.json
+
+run_bcg:
+	go run . -crawl=src
+
+go-unit-test:
+	go test ./...
+
+go-integration-test:
+	./integrations_tests/*.sh

test-crawler/integrations_tests/01_crawl_folder_build_catalogue.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+go run . -crawl=src
+# for FILE in "outputs/*"; do echo $FILE | cat $FILE; done
+if [ -z "$(ls -A outputs)" ]; then
+   echo "No expected files in output directory"
+   exit 1
+fi

test-crawler/main.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -29,8 +31,17 @@ func main() {
 
 	config := NewConfig()
 
-	crawlRepoBehaviorsAndSaveToJSON(config)
+	var crawl string
+	flag.StringVar(&crawl, "crawl", "behavior", "what should this crawler do? -crawl=src (source files), -crawl=behavior (test files)")
+	flag.Parse()
 
+	if crawl == "behavior" {
+		crawlRepoBehaviorsAndSaveToJSON(config)
+	} else if crawl == "src" {
+		if err := crawlRepoSourceCodeAndSaveToYaml(config); err != nil {
+			fmt.Println(err.Error())
+		}
+	}
 }
 
 func crawlRepoBehaviorsAndSaveToJSON(config Config) {
@@ -81,6 +92,22 @@ func crawlRepoBehaviorsAndSaveToJSON(config Config) {
 	Save(result, config.TestCrawlerOutputMode, config.TestCrawlerOutputDir, config.TestCrawlerIndentJSON)
 }
 
+func crawlRepoSourceCodeAndSaveToYaml(config Config) error {
+	ctx := context.Background()
+	for _, path := range config.BehaviorGenPaths {
+		systemMethods, err := crawlFolderForSystemMethods(path)
+		if err != nil {
+			return err
+		}
+
+		if err := makeYAMLFromSystemMethods(ctx, config, *systemMethods); err != nil {
+			return fmt.Errorf("error %w saving to yaml for system %s", err, systemMethods.System)
+		}
+	}
+
+	return nil
+}
+
 // crawlSingleFileForMethods accepts path of single go file,
 // and prints extracted methods out of it.
 func crawlSingleFileForFunctions(ctx context.Context, path string) ([]c.FunctionAnnotation, error) {
@@ -118,29 +145,34 @@ func crawlFolderForSystemMethods(system string) (*c.SystemMethods, error) {
 	return &systemFunctions, nil
 }
 
-// makeYAML will make yaml file from public methods.
-func makeYAML(ctx context.Context, filePath string) error {
+// makeYAMLFromSystemMethods will make yaml file from public methods.
+func makeYAMLFromSystemMethods(ctx context.Context, config Config, systemMethods c.SystemMethods) error {
 
-	publicMethods, err := crawlSingleFileForFunctions(ctx, filePath)
-	if err != nil {
-		fmt.Print(err)
-		os.Exit(1)
+	if _, err := os.Stat(config.BehaviorGenOutputDir); errors.Is(err, os.ErrNotExist) {
+		err2 := os.Mkdir(config.BehaviorGenOutputDir, os.ModePerm)
+		if err2 != nil {
+			return fmt.Errorf("error create dir: %w", err)
+		}
 	}
 
-	for i := 0; i < len(publicMethods); i++ {
-		publicMethods[i].ID = i
+	yamlData, err := y.Marshal(&systemMethods)
+	if err != nil {
+		return fmt.Errorf("error marhaling: %w", err)
 	}
 
-	yamlData, err := y.Marshal(&publicMethods)
+	filename := fmt.Sprintf("%s/%s.yaml", config.BehaviorGenOutputDir, systemMethods.System)
+	file, err := os.Create(filename)
 	if err != nil {
-		fmt.Printf("Error while Marshaling. %v", err)
+		return fmt.Errorf("error create file: %w", err)
 	}
+	defer file.Close()
+	fmt.Println("File generated: ", filename)
 
-	fileName := "test.yaml"
-	err = ioutil.WriteFile(fileName, yamlData, 0644)
+	err = ioutil.WriteFile(filename, yamlData, 0644)
 	if err != nil {
-		panic("Unable to write data into the file")
+		return fmt.Errorf("error write to file: %w", err)
 	}
+
 	return nil
 }