add time to test

Author: xixingde

deploy/weaviate-k8s.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: weaviate
+  namespace: costrict
+  labels:
+    app: weaviate
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: weaviate
+  template:
+    metadata:
+      labels:
+        app: weaviate
+    spec:
+      containers:
+      - name: weaviate
+        image: semitechnologies/weaviate:1.30.0
+        ports:
+        - containerPort: 8080
+          name: http
+        - containerPort: 50051
+          name: grpc
+        env:
+        - name: PERSISTENCE_DATA_PATH
+          value: "/var/lib/weaviate"
+        - name: QUERY_DEFAULTS_LIMIT
+          value: "-25"
+        - name: AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED
+          value: "true"
+        - name: DEFAULT_VECTORIZER_MODULE
+          value: "none"
+        - name: ASYNC_INDEXING
+          value: "true"
+        - name: AUTHENTICATION_APIKEY_ENABLED
+          value: "false"
+        volumeMounts:
+        - name: weaviate-data
+          mountPath: /var/lib/weaviate
+      volumes:
+      - name: weaviate-data
+        persistentVolumeClaim:
+          claimName: weaviate-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: weaviate-service
+  namespace: costrict
+  labels:
+    app: weaviate
+spec:
+  type: NodePort
+  ports:
+  - port: 8080
+    targetPort: 8080
+    protocol: TCP
+    name: http
+    nodePort: 32003
+  - port: 50051
+    targetPort: 50051
+    protocol: TCP
+    name: grpc
+    nodePort: 32004
+  selector:
+    app: weaviate
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: weaviate-pvc
+  namespace: costrict
+  labels:
+    app: weaviate
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: standard

internal/embedding/splitter.go

@@ -93,6 +93,7 @@ func (p *CodeSplitter) Split(codeFile *types.SourceFile) ([]*types.CodeChunk, er
 				allChunks = append(allChunks, subChunks...)
 			} else {
 				allChunks = append(allChunks, &types.CodeChunk{
+					Language:     "code",
 					CodebaseId:   codeFile.CodebaseId,
 					CodebasePath: codeFile.CodebasePath,
 					CodebaseName: codeFile.CodebaseName,
@@ -210,6 +211,7 @@ func (p *CodeSplitter) splitFuncWithSlidingWindow(content string, codeFile *type
 		endColumn := calculateColumn(content[startByte:endByte+1], endByte-startByte)
 
 		chunks = append(chunks, &types.CodeChunk{
+			Language:     "code",
 			CodebaseId:   codeFile.CodebaseId,
 			CodebasePath: codeFile.CodebasePath,
 			CodebaseName: codeFile.CodebaseName,
@@ -302,6 +304,7 @@ func (p *CodeSplitter) splitMarkdownFile(codeFile *types.SourceFile) ([]*types.C
 				tokenCount := p.countToken([]byte(chunkContent))
 
 				chunks = append(chunks, &types.CodeChunk{
+					Language:     "markdown",
 					CodebaseId:   codeFile.CodebaseId,
 					CodebasePath: codeFile.CodebasePath,
 					CodebaseName: codeFile.CodebaseName,
@@ -320,6 +323,7 @@ func (p *CodeSplitter) splitMarkdownFile(codeFile *types.SourceFile) ([]*types.C
 					tokenCount := p.countToken([]byte(chunkContent))
 
 					chunks = append(chunks, &types.CodeChunk{
+						Language:     "markdown",
 						CodebaseId:   codeFile.CodebaseId,
 						CodebasePath: codeFile.CodebasePath,
 						CodebaseName: codeFile.CodebaseName,
@@ -347,6 +351,7 @@ func (p *CodeSplitter) splitMarkdownFile(codeFile *types.SourceFile) ([]*types.C
 				tokenCount := p.countToken([]byte(chunkContent))
 
 				chunks = append(chunks, &types.CodeChunk{
+					Language:     "markdown",
 					CodebaseId:   codeFile.CodebaseId,
 					CodebasePath: codeFile.CodebasePath,
 					CodebaseName: codeFile.CodebaseName,
@@ -386,6 +391,7 @@ func (p *CodeSplitter) splitMarkdownFile(codeFile *types.SourceFile) ([]*types.C
 		tokenCount := p.countToken([]byte(chunkContent))
 
 		chunks = append(chunks, &types.CodeChunk{
+			Language:     "markdown",
 			CodebaseId:   codeFile.CodebaseId,
 			CodebasePath: codeFile.CodebasePath,
 			CodebaseName: codeFile.CodebaseName,