Fix logic for comment parse, resolve conflics

Author: StefanGajic

test-crawler/collector/collector.go

@@ -1,9 +1,12 @@
 package collector
 
 import (
+	"bytes"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode"
 
 	a "testsuites/annotations"
 )
@@ -126,3 +129,35 @@ func ListGoFilesInFolder(root string, ignore []string) (system string, files []s
 
 	return "", nil, nil
 }
+
+// GenerateMethodName generates name adding underscore and usecase index
+// ex: name: ListenProofRequest -> LISTEN_PROOF_REQUEST_001
+func GenerateMethodName(funcName string) string {
+	usecase := "001" // in the future it will be autoincremented depending how many return cases we have.
+	buf := &bytes.Buffer{}
+	for i, rune := range funcName {
+		if unicode.IsUpper(rune) && i > 0 {
+			buf.WriteRune('_')
+		}
+		buf.WriteRune(rune)
+	}
+
+	return fmt.Sprintf("%s_%s", strings.ToUpper(buf.String()), usecase)
+}
+
+// GenerateMethodDescription generates description based on inputParams, returnParams
+func GenerateMethodDescription(f FunctionAnnotation) string {
+	if f.Description == "" {
+		dsc := "Function description not set."
+		if (f.InputParams != "" && f.InputParams != "()") && f.ReturnValues != "" {
+			dsc = fmt.Sprintf(`Given a %s, returns %s`, f.InputParams, f.ReturnValues)
+		} else if (f.InputParams == "" || f.InputParams == "()") && f.ReturnValues != "" {
+			dsc = fmt.Sprintf(`Returns %s`, f.ReturnValues)
+		}
+		return dsc
+	} else {
+		return f.Description
+	}
+
+	return "Function description not set."
+}

test-crawler/config.go

@@ -15,12 +15,16 @@ const (
 )
 
 type Config struct {
-	Paths      []string   `yaml:"paths"`
-	OutputMode OutputMode `yaml:"output"`
-	OutputDir  string     `yaml:"output_dir"`
-	Language   string     `yaml:"lang_mode"`
-	IndentJSON bool       `yaml:"indent_json"`
-	Ignore     []string   `yaml:"ignore"`
+	TestCrawlerPaths      []string   `yaml:"test_crawler_paths"`
+	TestCrawlerOutputMode OutputMode `yaml:"test_crawler_output"`
+	TestCrawlerOutputDir  string     `yaml:"test_crawler_output_dir"`
+	TestCrawlerLanguage   string     `yaml:"test_crawler_lang_mode"`
+	TestCrawlerIndentJSON bool       `yaml:"test_crawler_indent_json"`
+	TestCrawlerIgnore     []string   `yaml:"test_crawler_ignore"`
+
+	BehaviorGenPaths     []string `yaml:"behavior_gen_paths"`
+	BehaviorGenIgnore    []string `yaml:"behavior_gen_ignore"`
+	BehaviorGenOutputDir string   `yaml:"behavior_gen_output_dir"`
 }
 
 const filename string = "config.yaml"
@@ -50,9 +54,9 @@ func NewConfig() Config {
 
 func Default() Config {
 	return Config{
-		Paths:      []string{"_modules/lotus"},
-		OutputMode: "stdout",
-		Language:   "auto",
-		IndentJSON: false,
+		TestCrawlerPaths:      []string{"_modules/lotus"},
+		TestCrawlerOutputMode: "stdout",
+		TestCrawlerLanguage:   "auto",
+		TestCrawlerIndentJSON: false,
 	}
 }

test-crawler/config.yaml

@@ -1,12 +1,17 @@
 # list of paths to directories that should be crawled
-paths: ["../repo-to-crawl"]
+test_crawler_paths: ["../repo-to-crawl"]
 # stdout or file(json)
-output: stdout
+test_crawler_output: stdout
 # if output is file, choose directory
-output_dir: outputs
+test_crawler_output_dir: outputs
 # auto or one of the supported languages for specific repo
-lang_mode: go
+test_crawler_lang_mode: go
 # should the output JSON be indented
-indent_json: true
+test_crawler_indent_json: true
 # ignore paths
-ignore: ["extern"]
+test_crawler_ignore: ["extern"]
+
+### Behavior catalogue related vars
+behavior_gen_paths: ["../repo-to-crawl"]
+behavior_gen_ignore: [".github","cmds","config","docker","docs","extern","integrate_test"]
+behavior_gen_output_dir: outputs/catalogue

test-crawler/config_test.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+func TestNewConfig(t *testing.T) {
+	cfg := Config{
+		TestCrawlerPaths:      []string{"a", "b"},
+		TestCrawlerOutputMode: "stdout",
+		TestCrawlerOutputDir:  "output",
+		TestCrawlerLanguage:   "go",
+		TestCrawlerIndentJSON: true,
+		TestCrawlerIgnore:     []string{"extern"},
+
+		BehaviorGenPaths:     []string{"c", "d"},
+		BehaviorGenIgnore:    []string{".git"},
+		BehaviorGenOutputDir: "output/catalogue",
+	}
+
+	f, err := os.CreateTemp("", "config-test.yml")
+	require.Nil(t, err)
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	configBytes, err := yaml.Marshal(&cfg)
+	assert.NoError(t, err)
+
+	assert.NoError(t, ioutil.WriteFile(f.Name(), configBytes, 0644))
+
+	var configFromFile Config
+	fileContent, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+
+	err = yaml.Unmarshal(fileContent, &configFromFile)
+	assert.NoError(t, err)
+
+	assert.Equal(t, cfg, configFromFile)
+}

test-crawler/crawler_test.go

@@ -22,22 +22,23 @@ func TestCrawlSingleFileForFunctions(t *testing.T) {
 	}
 
 	assert.Equal(t, "// HelloEvent simple method that just formats message.", fnsAnn[0].Description)
-	assert.Equal(t, "HelloEvent", fnsAnn[0].Name)
+	assert.Equal(t, "HELLO_EVENT_001", fnsAnn[0].Name)
 	assert.Equal(t, "()", fnsAnn[0].InputParams)      // input param
 	assert.Equal(t, "string", fnsAnn[0].ReturnValues) // return param
 
 	assert.Equal(t, "// HelloEventWithParameter accepts one param that got formated in message.", fnsAnn[1].Description)
-	assert.Equal(t, "HelloEventWithParameter", fnsAnn[1].Name)
+	assert.Equal(t, "HELLO_EVENT_WITH_PARAMETER_001", fnsAnn[1].Name)
 	assert.Equal(t, "(param string)", fnsAnn[1].InputParams)
 	assert.Equal(t, "(string, error)", fnsAnn[1].ReturnValues)
 
 	assert.Equal(t, "// FunctionWithoutParameters...", fnsAnn[2].Description)
-	assert.Equal(t, "FunctionWithoutParameters", fnsAnn[2].Name)
+	assert.Equal(t, "FUNCTION_WITHOUT_PARAMETERS_001", fnsAnn[2].Name)
 	assert.Equal(t, "()", fnsAnn[2].InputParams)
 	assert.Equal(t, "", fnsAnn[2].ReturnValues)
 
 	assert.Equal(t, "// FunctionWithPointerReturnValue returns a simple pointer value.", fnsAnn[3].Description)
-	assert.Equal(t, "FunctionWithPointerReturnValue", fnsAnn[3].Name)
+	assert.Equal(t, "FUNCTION_WITH_POINTER_RETURN_VALUE_001", fnsAnn[3].Name)
+
 	assert.Equal(t, "()", fnsAnn[3].InputParams)
 	assert.Equal(t, "*Event", fnsAnn[3].ReturnValues)
 }

test-crawler/extractor/extractor.go

@@ -260,14 +260,14 @@ func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.P
 				child.Child(4): parameter_list:  (<-chan *types2.RequestEvent, error)
 			*/
 			if child.Type() == string(METHOD_DECLARATION) {
-				comment := content[prevNode.StartByte():prevNode.EndByte()]
+				comment := ""
 				if prevNode.Type() == string(COMMENT) {
+					comment = content[prevNode.StartByte():prevNode.EndByte()]
 					value, annotationType, _ := parser.Parse(comment)
 					if value != nil && annotationType == a.Ignore {
 						isIgnored = value.(bool)
 					}
 				}
-
 				funcName = content[child.Child(2).StartByte():child.Child(2).EndByte()]
 				params := ""
 				returnValues := ""
@@ -286,15 +286,19 @@ func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.P
 					returnValueType == string(TYPE_IDENTIFIER) {
 					returnValues = content[child.Child(4).StartByte():child.Child(4).EndByte()]
 				}
+				funcAnn := c.FunctionAnnotation{
+					Description:  comment,
+					Name:         funcName,
+					Public:       isPublic(funcName),
+					InputParams:  params,
+					ReturnValues: returnValues,
+				}
+				funcAnn.Name = c.GenerateMethodName(funcAnn.Name)
+				funcAnn.Description = c.GenerateMethodDescription(funcAnn)
+
 				funcAnnoPair = append(funcAnnoPair, FunctionAnnotationNode{
-					Node: child,
-					Function: c.FunctionAnnotation{
-						Description:  comment,
-						Name:         funcName,
-						Public:       isPublic(funcName),
-						InputParams:  params,
-						ReturnValues: returnValues,
-					},
+					Node:     child,
+					Function: funcAnn,
 				})
 			}
 			prevNode = child

test-crawler/main.go

@@ -37,8 +37,8 @@ func crawlRepoBehaviorsAndSaveToJSON(config Config) {
 	allFiles := make(map[c.FileID]*c.TestFile)
 	var fileFunctions []c.Function
 
-	for _, path := range config.Paths {
-		files, err := c.GetTestFiles(path, config.Ignore)
+	for _, path := range config.TestCrawlerPaths {
+		files, err := c.GetTestFiles(path, config.TestCrawlerIgnore)
 		if err != nil {
 			fmt.Println(err)
 			return
@@ -78,7 +78,7 @@ func crawlRepoBehaviorsAndSaveToJSON(config Config) {
 
 	result := filterFilesWhereChildIsRoot(allFiles)
 
-	Save(result, config.OutputMode, config.OutputDir, config.IndentJSON)
+	Save(result, config.TestCrawlerOutputMode, config.TestCrawlerOutputDir, config.TestCrawlerIndentJSON)
 }
 
 // crawlSingleFileForMethods accepts path of single go file,