Merge branch 'main' of github.com:zgsm-ai/codebase-embedder

Author: wantWhatBike

.github/workflows/build.yml

@@ -4,6 +4,10 @@ on:
   push:
     tags:
       - 'v*.*.*'
+    branches:
+      - main
+      - master
+      - feature/tree-sitter-markdown
 
 jobs:
   build-and-push:
@@ -15,6 +19,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          fetch-tags: true
+          force: true
 
       - name: Setup Node.js
         uses: actions/setup-node@v3
@@ -53,9 +59,12 @@ jobs:
       - name: Release
         run: |
           yarn global add semantic-release@17.4.4
-          semantic-release --branches main,master
+          # 清理可能冲突的本地tags
+          git fetch --tags --force
+          semantic-release --branches main,master,feature/tree-sitter-markdown --no-ci
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

.gitignore

@@ -31,4 +31,6 @@ bin/*
 docs/task.md
 cmd.sh
 testfiles
-.history
+.history
+.coignore
+.costrict

deploy/embber-debug.yaml

@@ -34,6 +34,7 @@ data:
         OverlapTokens: 100
         MaxTokensPerChunk: 1000
         EnableMarkdownParsing: false
+        EnableOpenAPIParsing: false
       GraphTask:
         MaxConcurrency: 100
         Timeout: 18000s

deploy/embber.yaml

@@ -34,6 +34,7 @@ data:
         OverlapTokens: 100
         MaxTokensPerChunk: 1000
         EnableMarkdownParsing: false
+        EnableOpenAPIParsing: false
       GraphTask:
         MaxConcurrency: 100
         Timeout: 18000s

etc/conf.yaml

@@ -27,7 +27,8 @@ IndexTask:
     Timeout: 300s
     OverlapTokens: 100
     MaxTokensPerChunk: 1000
-    EnableMarkdownParsing: true
+    EnableMarkdownParsing: false
+    EnableOpenAPIParsing: false
   GraphTask:
     MaxConcurrency: 10
     Timeout: 300s

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/tiktoken-go/tokenizer v0.6.2
 	github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0
+	github.com/tree-sitter-grammars/tree-sitter-markdown v0.5.0
 	github.com/tree-sitter/go-tree-sitter v0.25.0
 	github.com/tree-sitter/tree-sitter-c v0.24.1
 	github.com/tree-sitter/tree-sitter-c-sharp v0.23.1

go.sum

@@ -320,6 +320,8 @@ github.com/tiktoken-go/tokenizer v0.6.2 h1:t0GN2DvcUZSFWT/62YOgoqb10y7gSXBGs0A+4
 github.com/tiktoken-go/tokenizer v0.6.2/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w=
 github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0 h1:SWIUDASa+WPhDDem1U5IJpYwQEezkqXrUI61OcnORzM=
 github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0/go.mod h1:eH+flFf3QOa9c9BY9g3Bz02F7zTq30kGIG3cgB0lSlI=
+github.com/tree-sitter-grammars/tree-sitter-markdown v0.5.0 h1:93UWmJmfHbEiYmWF75ou2GE0fNlm0RwXRga3NGOCTEM=
+github.com/tree-sitter-grammars/tree-sitter-markdown v0.5.0/go.mod h1:Cw6XOdJRZZt7RKnDszrMJwsfTL+2mQRiz+nlE694HNY=
 github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU=
 github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU=
 github.com/tree-sitter/tree-sitter-c v0.24.1 h1:GV9DjvIV6uYe3W/JBKMFwE4hJcRxzRDq63llxNFHOkY=

internal/config/index.go

@@ -22,6 +22,7 @@ type EmbeddingTaskConf struct {
 	OverlapTokens         int
 	MaxTokensPerChunk     int
 	EnableMarkdownParsing bool `json:",default=false"` // 是否启用markdown文件解析
+	EnableOpenAPIParsing  bool `json:",default=false"` // 是否启用OpenAPI文档解析
 }
 
 type GraphTaskConf struct {