VEN-1 | Parse single file for function declarations (#943)

* VEN-1 | Parse single file for function declarations

Author: bokimilinkovic

test-crawler/collector/collector.go

@@ -28,6 +28,14 @@ type Function struct {
 	IsTesting       bool `json:"-"`
 }
 
+type FunctionAnnotation struct {
+	Name         string `yaml:"name"`
+	InputParams  string `yaml:"inputParams"`
+	ReturnValues string `yaml:"returnValues"`
+	Description  string
+	Public       bool
+}
+
 func GetTestFiles(root string, ignore []string) (files []TestFile, err error) {
 	fileArray, err := listTestFiles(root, ignore)
 	if err != nil {

test-crawler/crawler_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtractPublicMethodsFromFile(t *testing.T) {
+	fnsAnn, err := extractPublicMethodsFromFile(context.Background(), "./mocks/event.go")
+	if err != nil {
+		t.Errorf("got error: %v", err.Error())
+	}
+
+	if len(fnsAnn) != 2 {
+		t.Errorf("got %q, expected %q methods", len(fnsAnn), 2)
+	}
+
+	assert.Equal(t, "HelloEvent", fnsAnn[0].Name)
+	assert.Equal(t, "()", fnsAnn[0].InputParams)      // input param
+	assert.Equal(t, "string", fnsAnn[0].ReturnValues) // return param
+
+	assert.Equal(t, "HelloEventWithParameter", fnsAnn[1].Name)
+	assert.Equal(t, "(param string)", fnsAnn[1].InputParams)
+	assert.Equal(t, "(string, error)", fnsAnn[1].ReturnValues)
+}

test-crawler/extractor/extractor.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"unicode"
 
 	a "testsuites/annotations"
 	c "testsuites/collector"
@@ -26,6 +27,7 @@ const (
 	BLOCK                NodeType = "block"
 	CALL_EXPRESSION      NodeType = "call_expression"
 	IDENTIFIER           NodeType = "identifier"
+	METHOD_DECLARATION   NodeType = "method_declaration"
 )
 
 type Metadata struct {
@@ -39,6 +41,11 @@ type FileData struct {
 	Functions []c.Function
 }
 
+type FunctionAnnotationNode struct {
+	Node     *sitter.Node
+	Function c.FunctionAnnotation
+}
+
 type hasNoTests bool
 
 func ExtractInfo(file c.TestFile, ctx context.Context, fileID c.FileID) (*FileData, hasNoTests, error) {
@@ -81,6 +88,29 @@ func ExtractInfo(file c.TestFile, ctx context.Context, fileID c.FileID) (*FileDa
 	return fileData, hasNoTests(!checkForExistanceOfTests(fData)), nil
 }
 
+func GetExportedFunctions(ctx context.Context, filePath string) ([]c.FunctionAnnotation, error) {
+	content, err := getFileContent(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	parser := sitter.NewParser()
+	parser.SetLanguage(golang.GetLanguage())
+
+	tree, err := parser.ParseCtx(ctx, nil, []byte(content))
+	if err != nil {
+		return nil, err
+	}
+
+	cursor := sitter.NewTreeCursor(tree.RootNode())
+	fnsAnno, err := parseContentForFunctions(content, cursor)
+	if err != nil {
+		return nil, err
+	}
+
+	return fnsAnno, nil
+}
+
 func getFileContent(filePath string) (content string, err error) {
 
 	file, err := os.Open(filePath)
@@ -103,17 +133,17 @@ func parseContent(content string, treeCursor *sitter.TreeCursor, filePath string
 	var annotationParser a.Parser
 
 	fileData.Metadata = getMetadata(content, treeCursor, &annotationParser)
-	functions := getFunctionNodes(content, treeCursor, &annotationParser)
+	funcNodes := getFunctionNodes(content, treeCursor, &annotationParser)
 
-	for _, function := range functions {
+	for _, funcNode := range funcNodes {
 
-		behaviors := findBehaviorsFromNode(content, function.Node)
+		behaviors := findBehaviorsFromNode(content, funcNode.Node)
 		var callExpressions []string = nil
 		if len(behaviors) > 0 {
-			callExpressions = findCallExprFromNode(content, function.Node)
+			callExpressions = findCallExprFromNode(content, funcNode.Node)
 		}
 
-		fileData.Functions = append(fileData.Functions, makeCollectorScenario(filePath, function.Name, behaviors, callExpressions))
+		fileData.Functions = append(fileData.Functions, makeCollectorScenario(filePath, funcNode.Function.Name, behaviors, callExpressions))
 
 	}
 
@@ -163,10 +193,11 @@ func getMetadata(content string, treeCursor *sitter.TreeCursor, parser *a.Parser
 	return &meta
 }
 
-func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.Parser) (funcAnnoPair []struct {
-	Node *sitter.Node
-	Name string
-}) {
+// getFunctionNodes method extracts functions and methods.
+// Function can have 2 types: function_declaration (for example contructor)
+// and method_declaration (can be exported and unexported).
+// Return value is a slice of FunctionAnnotationNode, where each node holds function's annotation.
+func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.Parser) (funcAnnoPair []FunctionAnnotationNode) {
 
 	numChildsRootNode := treeCursor.CurrentNode().ChildCount()
 	node := &sitter.Node{}
@@ -203,18 +234,48 @@ func getFunctionNodes(content string, treeCursor *sitter.TreeCursor, parser *a.P
 					continue
 				}
 
-				funcAnnoPair = append(funcAnnoPair, struct {
-					Node *sitter.Node
-					Name string
-				}{
+				funcAnnoPair = append(funcAnnoPair, FunctionAnnotationNode{
 					Node: node,
-					Name: funcName,
+					Function: c.FunctionAnnotation{
+						Name: funcName,
+					},
 				})
 
 				node = nil
 				funcName = ""
 				isIgnored = false
 			}
+			/*
+				Take a look at this function for example:
+				*********
+				func (e *ProofEventStream) ListenProofEvent(
+					ctx context.Context,
+					policy *types2.ProofRegisterPolicy)
+				(<-chan *types2.RequestEvent, error){...}
+				*********
+				child.Child(1) is a first part of function declaration (e *ProofEventStream)
+				child.Child(2): field_identifier  'ListenProofEvent'
+				child.Child(3): parameter_list:  (ctx context.Context, policy *types2.ProofRegisterPolicy)
+				child.Child(4): parameter_list:  (<-chan *types2.RequestEvent, error)
+			*/
+			if child.Type() == string(METHOD_DECLARATION) {
+				funcName = content[child.Child(2).StartByte():child.Child(2).EndByte()]
+				params := ""
+				returnValues := ""
+				if child.Child(1).Type() == string(PARAMETER_LIST) {
+					params = content[child.Child(3).StartByte():child.Child(3).EndByte()]
+					returnValues = content[child.Child(4).StartByte():child.Child(4).EndByte()]
+				}
+				funcAnnoPair = append(funcAnnoPair, FunctionAnnotationNode{
+					Node: child,
+					Function: c.FunctionAnnotation{
+						Name:         funcName,
+						Public:       isPublic(funcName),
+						InputParams:  params,
+						ReturnValues: returnValues,
+					},
+				})
+			}
 			prevNode = child
 		}
 	}
@@ -301,3 +362,21 @@ func makeID(filePath string, funcName string, behavior string) string {
 	hash := md5.Sum([]byte(fmt.Sprintf("%s_%s_%s", filePath, funcName, behavior)))
 	return string(hex.EncodeToString(hash[:]))
 }
+
+// parseContentForFunctions accepts a content which represents whole golang file as a string,
+// parses it and returns a slice of function annotations (including exported and unexported ones).
+func parseContentForFunctions(content string, cursor *sitter.TreeCursor) ([]c.FunctionAnnotation, error) {
+	var annotationParser a.Parser
+
+	var fnsAnno []c.FunctionAnnotation
+	funcNodes := getFunctionNodes(content, cursor, &annotationParser)
+	for _, funcNode := range funcNodes {
+		fnsAnno = append(fnsAnno, funcNode.Function)
+	}
+
+	return fnsAnno, nil
+}
+
+func isPublic(funcName string) bool {
+	return unicode.IsUpper(rune(funcName[0]))
+}

test-crawler/go.mod

@@ -2,6 +2,13 @@ module testsuites
 
 go 1.17
 
-require github.com/smacker/go-tree-sitter v0.0.0-20211116060328-db7fde9b5e82
+require (
+	github.com/smacker/go-tree-sitter v0.0.0-20211116060328-db7fde9b5e82
+	github.com/stretchr/testify v1.4.0
+)
 
-require gopkg.in/yaml.v2 v2.4.0 // indirect
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
+)

test-crawler/go.sum

@@ -7,8 +7,8 @@ github.com/smacker/go-tree-sitter v0.0.0-20211116060328-db7fde9b5e82/go.mod h1:E
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

test-crawler/main.go

@@ -26,6 +26,10 @@ func main() {
 
 	config := NewConfig()
 
+	crawlRepoBehaviorsAndSaveToJSON(config)
+}
+
+func crawlRepoBehaviorsAndSaveToJSON(config Config) {
 	allFiles := make(map[c.FileID]*c.TestFile)
 	var fileFunctions []c.Function
 
@@ -73,6 +77,29 @@ func main() {
 	Save(result, config.OutputMode, config.OutputDir, config.IndentJSON)
 }
 
+// crawlSingleFileForMethods accepts path of single go file,
+// and prints extracted methods out of it.
+func crawlSingleFileForMethods(path string) {
+	fns, err := extractPublicMethodsFromFile(context.Background(), path)
+	if err != nil {
+		fmt.Print(err)
+		os.Exit(1)
+	}
+	for _, fn := range fns {
+		fmt.Printf(
+			"name %s : public %v : params: %s : return values : %s\n",
+			fn.Name,
+			fn.Public,
+			fn.InputParams,
+			fn.ReturnValues,
+		)
+	}
+}
+
+func extractPublicMethodsFromFile(ctx context.Context, filePath string) ([]c.FunctionAnnotation, error) {
+	return ex.GetExportedFunctions(ctx, filePath)
+}
+
 func linkFiles(flist []c.Function) (links [][]FnLink) {
 
 	functions := make(map[string]c.Function)

test-crawler/mocks/event.go

@@ -0,0 +1,31 @@
+package mocks
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+type Event struct {
+	ID string `json:"id"`
+}
+
+func NewEvent() *Event {
+	return &Event{
+		ID: time.Now().String(),
+	}
+}
+
+// HelloEvent simple method that just formats message.
+func (e *Event) HelloEvent() string {
+	return fmt.Sprintf("Simple HelloEvent")
+}
+
+// HelloEventWithParameter accepts one param that got formated in message.
+func (e *Event) HelloEventWithParameter(param string) (string, error) {
+	if param == "" {
+		return "", errors.New("no param provided")
+	}
+
+	return fmt.Sprintf("HelloEventWithParameter: %v", param), nil
+}