VEN-4 Add system parsing, support different types of return values, w… (#944)

* VEN-4 Add system parsing, support different types of return values, write new test cases

Author: bokimilinkovic

test-crawler/collector/collector.go

@@ -37,6 +37,11 @@ type FunctionAnnotation struct {
 	Public       bool
 }
 
+type SystemMethods struct {
+	System  string               `yaml:"system"`
+	Methods []FunctionAnnotation `yaml:"methods"`
+}
+
 func GetTestFiles(root string, ignore []string) (files []TestFile, err error) {
 	fileArray, err := listTestFiles(root, ignore)
 	if err != nil {
@@ -89,3 +94,35 @@ func listTestFiles(root string, ignore []string) (files []string, err error) {
 
 	return nil, nil
 }
+
+// listGoFilesInFolder returns list of all .go files, excluding _test.go
+// Root folder will represent a system for which the methods will be crawled.
+func ListGoFilesInFolder(root string, ignore []string) (system string, files []string, err error) {
+	err = filepath.Walk(root,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			// If it's directory, get the name of system.
+			if root == path && info.IsDir() {
+				system = info.Name()
+			}
+
+			if strings.HasSuffix(path, ".go") && !strings.Contains(path, "_test") {
+				files = append(files, path)
+			}
+
+			return nil
+		})
+
+	if err != nil {
+		return "", nil, err
+	}
+
+	if len(files) != 0 {
+		return system, files, nil
+	}
+
+	return "", nil, nil
+}

test-crawler/collector/collector_test.go

@@ -0,0 +1,43 @@
+package collector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListGoFilesInFolder(t *testing.T) {
+	t.Run("Read golang files from temporary directory", func(t *testing.T) {
+		tempDir := t.TempDir()
+		err := ioutil.WriteFile(fmt.Sprintf("%s/a.go", tempDir), []byte("package main func main(){a:=1}"), 0644)
+		assert.NoError(t, err)
+		err = ioutil.WriteFile(fmt.Sprintf("%s/b.go", tempDir), []byte("package main func main(){b:=2}"), 0644)
+		assert.NoError(t, err)
+
+		system, files, err := ListGoFilesInFolder(tempDir, nil)
+		assert.NoError(t, err)
+		splitedFolderName := strings.Split(tempDir, "/")
+		systemName := splitedFolderName[len(splitedFolderName)-1]
+		assert.Equal(t, system, systemName)
+		assert.Equal(t, 2, len(files))
+	})
+
+	t.Run("Read golang files excluding everything else", func(t *testing.T) {
+		tempDir := t.TempDir()
+		err := ioutil.WriteFile(fmt.Sprintf("%s/a.go", tempDir), []byte("package main func main(){a:=1}"), 0644)
+		err = ioutil.WriteFile(fmt.Sprintf("%s/b.go", tempDir), []byte("package main func main(){b:=2}"), 0644)
+		err = ioutil.WriteFile(fmt.Sprintf("%s/b_test.go", tempDir), []byte("package main_test"), 0644)
+		err = ioutil.WriteFile(fmt.Sprintf("%s/.env", tempDir), []byte("PORT=8080"), 0644)
+		assert.NoError(t, err)
+
+		system, files, err := ListGoFilesInFolder(tempDir, nil)
+		assert.NoError(t, err)
+		splitedFolderName := strings.Split(tempDir, "/")
+		systemName := splitedFolderName[len(splitedFolderName)-1]
+		assert.Equal(t, system, systemName)
+		assert.Equal(t, 2, len(files))
+	})
+}

test-crawler/crawler_test.go

@@ -2,18 +2,22 @@ package main
 
 import (
 	"context"
+	"io/ioutil"
 	"testing"
+	c "testsuites/collector"
 
 	"github.com/stretchr/testify/assert"
+	y "gopkg.in/yaml.v2"
 )
 
-func TestExtractPublicMethodsFromFile(t *testing.T) {
-	fnsAnn, err := extractPublicMethodsFromFile(context.Background(), "./mocks/event.go")
+func TestCrawlSingleFileForFunctions(t *testing.T) {
+	ctx := context.Background()
+	fnsAnn, err := crawlSingleFileForFunctions(ctx, "./mocks/event.go")
 	if err != nil {
 		t.Errorf("got error: %v", err.Error())
 	}
 
-	if len(fnsAnn) != 2 {
+	if len(fnsAnn) != 4 {
 		t.Errorf("got %q, expected %q methods", len(fnsAnn), 2)
 	}
 
@@ -24,4 +28,52 @@ func TestExtractPublicMethodsFromFile(t *testing.T) {
 	assert.Equal(t, "HelloEventWithParameter", fnsAnn[1].Name)
 	assert.Equal(t, "(param string)", fnsAnn[1].InputParams)
 	assert.Equal(t, "(string, error)", fnsAnn[1].ReturnValues)
+
+	assert.Equal(t, "FunctionWithoutParameters", fnsAnn[2].Name)
+	assert.Equal(t, "()", fnsAnn[2].InputParams)
+	assert.Equal(t, "", fnsAnn[2].ReturnValues)
+
+	assert.Equal(t, "FunctionWithPointerReturnValue", fnsAnn[3].Name)
+	assert.Equal(t, "()", fnsAnn[3].InputParams)
+	assert.Equal(t, "*Event", fnsAnn[3].ReturnValues)
+}
+
+func TestMakeYAML(t *testing.T) {
+
+	yaml := []c.FunctionAnnotation{
+		{
+			ID:           1,
+			Name:         "SomeName",
+			InputParams:  "(ctx context.Context, param Parameters)",
+			ReturnValues: "error",
+			Description:  "",
+			Public:       true,
+		},
+		{
+			ID:           2,
+			Name:         "SomeName2",
+			InputParams:  "(ctx context.Context, param2 Parameters2)",
+			ReturnValues: "error",
+			Description:  "",
+			Public:       true,
+		},
+	}
+
+	yamlData, err := y.Marshal(&yaml)
+	if err != nil {
+		t.Errorf("got error: %v", err.Error())
+	}
+	fileName := "test.yaml"
+	err = ioutil.WriteFile(fileName, yamlData, 0644)
+	if err != nil {
+		panic("Unable to write data into the file")
+	}
+
+	bytes, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Errorf("got error: %v", err.Error())
+	}
+
+	assert.Equal(t, yamlData, bytes)
+
 }

test-crawler/extractor/extractor.go

@@ -28,6 +28,8 @@ const (
 	CALL_EXPRESSION      NodeType = "call_expression"
 	IDENTIFIER           NodeType = "identifier"
 	METHOD_DECLARATION   NodeType = "method_declaration"
+	POINTER_TYPE         NodeType = "pointer_type"
+	TYPE_IDENTIFIER      NodeType = "type_identifier"
 )
 
 type Metadata struct {
@@ -262,8 +264,19 @@ func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.P
 				funcName = content[child.Child(2).StartByte():child.Child(2).EndByte()]
 				params := ""
 				returnValues := ""
-				if child.Child(1).Type() == string(PARAMETER_LIST) {
+				if child.Child(3).Type() == string(PARAMETER_LIST) {
 					params = content[child.Child(3).StartByte():child.Child(3).EndByte()]
+				}
+				returnValueType := child.Child(4).Type()
+				/* Return value can be in one of these 3 forms
+				- (type1, type2) ->  PARAMETER_LIST
+				- *type1 -> POINTER_TYPE
+				- type1 ->  TYPE_IDENTIFIER
+				that's why we need to check all of these conditions.
+				*/
+				if returnValueType == string(PARAMETER_LIST) ||
+					returnValueType == string(POINTER_TYPE) ||
+					returnValueType == string(TYPE_IDENTIFIER) {
 					returnValues = content[child.Child(4).StartByte():child.Child(4).EndByte()]
 				}
 				funcAnnoPair = append(funcAnnoPair, FunctionAnnotationNode{

test-crawler/main.go

@@ -83,32 +83,45 @@ func crawlRepoBehaviorsAndSaveToJSON(config Config) {
 
 // crawlSingleFileForMethods accepts path of single go file,
 // and prints extracted methods out of it.
-func crawlSingleFileForMethods(path string) {
-	fns, err := extractPublicMethodsFromFile(context.Background(), path)
+func crawlSingleFileForFunctions(ctx context.Context, path string) ([]c.FunctionAnnotation, error) {
+	fns, err := ex.GetExportedFunctions(ctx, path)
 	if err != nil {
-		fmt.Print(err)
-		os.Exit(1)
-	}
-	for _, fn := range fns {
-		fmt.Printf(
-			"ID %d : name %s : public %v : params: %s : return values : %s\n",
-			fn.ID,
-			fn.Name,
-			fn.Public,
-			fn.InputParams,
-			fn.ReturnValues,
-		)
+		return nil, fmt.Errorf("error extracting public methods from file: %w", err)
 	}
+
+	return fns, nil
 }
 
-func extractPublicMethodsFromFile(ctx context.Context, filePath string) ([]c.FunctionAnnotation, error) {
-	return ex.GetExportedFunctions(ctx, filePath)
+// crawlFolderForSystemMethods accepts folder, which represents a system that will be crawled for
+// all of his methods.
+// ex: "../repo-to-crawl/venus-gateway/proofevent"
+func crawlFolderForSystemMethods(system string) (*c.SystemMethods, error) {
+	system, files, err := c.ListGoFilesInFolder(system, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	systemFunctions := c.SystemMethods{
+		System:  system,
+		Methods: make([]c.FunctionAnnotation, 0),
+	}
+
+	ctx := context.Background()
+	for _, file := range files {
+		fns, err := crawlSingleFileForFunctions(ctx, file)
+		if err != nil {
+			return nil, fmt.Errorf("can not crawl file: %s, err: %w", file, err)
+		}
+		systemFunctions.Methods = append(systemFunctions.Methods, fns...)
+	}
+
+	return &systemFunctions, nil
 }
 
 // makeYAML will make yaml file from public methods.
 func makeYAML(ctx context.Context, filePath string) error {
 
-	publicMethods, err := extractPublicMethodsFromFile(ctx, filePath)
+	publicMethods, err := crawlSingleFileForFunctions(ctx, filePath)
 	if err != nil {
 		fmt.Print(err)
 		os.Exit(1)

test-crawler/make_yaml_test.go

@@ -29,3 +29,13 @@ func (e *Event) HelloEventWithParameter(param string) (string, error) {
 
 	return fmt.Sprintf("HelloEventWithParameter: %v", param), nil
 }
+
+// FunctionWithoutParameters...
+func (e *Event) FunctionWithoutParameters() {
+
+}
+
+// FunctionWithPointerReturnValue returns a simple pointer value.
+func (e *Event) FunctionWithPointerReturnValue() *Event {
+	return &Event{}
+}