Merge pull request #6 from Robbings/dev

实现大模型支持，并完善配置文档

Author: mimo-x

README.md

@@ -54,6 +54,89 @@
   <img src="doc/img/img.png" style="width:500px;"/>
 </p>
 
+### 3. 更多种大模型接入
+1. 可通过实现项目接口快速接入自定义模型，具体配置方式参见[config.md](doc/config.md)。
+2. 项目通过[UnionLLM](https://github.com/EvalsOne/UnionLLM/)进行多模型支持，
+兼容[LiteLLM](https://docs.litellm.ai/docs)，默认支持模型如下表所示。
+
+<table style="width:100%; text-align:center; border-collapse:collapse;">
+  <tr>
+    <td>OpenAI</td>
+    <td>Azure</td>
+    <td>AWS - SageMaker</td>
+    <td>AWS - Bedrock</td>
+  </tr>
+  <tr>
+    <td>Google - Vertex_AI</td>
+    <td>Google - Palm</td>
+    <td>Google AI Studio - Gemini</td>
+    <td>Mistral AI API</td>
+  </tr>
+  <tr>
+    <td>Cloudflare AI Workers</td>
+    <td>Cohere</td>
+    <td>Anthropic</td>
+    <td>Empower</td>
+  </tr>
+  <tr>
+    <td>Huggingface</td>
+    <td>Replicate</td>
+    <td>Together_AI</td>
+    <td>OpenRouter</td>
+  </tr>
+  <tr>
+    <td>AI21</td>
+    <td>Baseten</td>
+    <td>Vllm</td>
+    <td>NLP_Cloud</td>
+  </tr>
+  <tr>
+    <td>Aleph Alpha</td>
+    <td>Petals</td>
+    <td>Ollama</td>
+    <td>Deepinfra</td>
+  </tr>
+  <tr>
+    <td>Perplexity-AI</td>
+    <td>Groq AI</td>
+    <td>DeepSeek</td>
+    <td>Anyscale</td>
+  </tr>
+  <tr>
+    <td>IBM - Watsonx.ai</td>
+    <td>Voyage AI</td>
+    <td>Xinference [Xorbits Inference]</td>
+    <td>FriendliAI</td>
+  </tr>
+  <tr>
+    <td>Galadriel</td>
+    <td>智谱AI</td>
+    <td>月之暗面 Moonshot</td>
+    <td>百度文心一言</td>
+  </tr>
+  <tr>
+    <td>阿里巴巴通义千问</td>
+    <td>MiniMax</td>
+    <td>讯飞星火</td>
+    <td>百川智能</td>
+  </tr>
+  <tr>
+    <td>昆仑天工</td>
+    <td>零一万物</td>
+    <td>阶跃星辰</td>
+    <td>字节豆包</td>
+  </tr>
+  <tr>
+    <td>深度求索 DeepSeek</td>
+    <td>More</td>
+    <td></td>
+    <td></td>
+  </tr>
+</table>
+
+
+
+
 
 # 部署 📖
 
@@ -70,6 +153,7 @@ pip install -r requirements.txt
 ```bash
 vim config/config.py
 ```
+
 4.**运行**
 ```bash
 python3 app.py
@@ -94,6 +178,7 @@ todo dockerfile
 # 待办清单 📌
 
 - ✅ 使用 GPT 进行Code Review
+- ✅ 实现多模型支持
 - [ ] 尝试接入私有化大模型解决代码安全问题
 - [ ] 可以配置更多的触发方式
   - ✅ Merge Request
@@ -127,4 +212,6 @@ todo dockerfile
 **This project is licensed under the [MIT License](https://chat.openai.com/c/9be6b422-f10c-4379-b152-e756230d54f8#:~:text=%E7%9A%84%E5%AE%8C%E6%95%B4%E6%96%87%E6%9C%AC%EF%BC%9A-,MIT%20License,-%E4%BD%A0%E5%8F%AF%E4%BB%A5%E8%AE%BF%E9%97%AE).**
 
 
+## Star History
 
+[![Star History Chart](https://api.star-history.com/svg?repos=mimo-x/Code-Review-GPT-Gitlab&type=Date)](https://star-history.com/#mimo-x/Code-Review-GPT-Gitlab&Date)

app.py

@@ -6,7 +6,7 @@
 app = Flask(__name__)
 app.config['debug'] = True
 
-# 路由组
+# router group
 app.register_blueprint(git, url_prefix='/git')
 
 

app/gitlab_utils.py

@@ -12,18 +12,16 @@ def get_merge_request_id(branch_name, project_id):
     :param project_id: 项目id
     :return: 如果分支存在 mr 则返回mrid / 如果不存在mr 则返回 ""
     """
-    # 构建API请求URL
     url = f"{gitlab_server_url}/api/v4/projects/{project_id}/merge_requests"
 
-    # 发送API请求，检查是否有与分支相关的Merge Request
     params = {
         "source_branch": branch_name,
         "state": "opened"  # 可以根据需求选择合适的状态（opened、closed、merged等）
     }
+    
     headers = {"Private-Token": gitlab_private_token}
     response = requests.get(url, params=params, headers=headers)
 
-    # 解析JSON响应并检查是否有相关的Merge Request
     if response.status_code == 200:
         merge_requests = response.json()
         if len(merge_requests) > 0:
@@ -38,27 +36,19 @@ def get_merge_request_id(branch_name, project_id):
 
 @retry(stop_max_attempt_number=3, wait_fixed=2000)
 def get_commit_list(merge_request_iid, project_id):
-    # Create API URL for the merge request commits
     api_url = f"{gitlab_server_url}/api/v4/projects/{project_id}/merge_requests/{merge_request_iid}/commits"
-    # Set the private token in the header
+    
     headers = {"PRIVATE-TOKEN": gitlab_private_token}
-
-    # Make a GET request to the API URL
     response = requests.get(api_url, headers=headers)
+
     commit_list = []
-    # If the response code is 200, the API call was successful
     if response.status_code == 200:
-        # Get the commits from the response
         commits = response.json()
-        # Iterate through the commits and print the commit ID and message
         for commit in commits:
             print(f"Commit ID: {commit['id']}, Message: {commit['message']}")
-            # Append the commit ID to the list
             commit_list.append(commit['id'])
     else:
-        # Log an error if the API call was unsuccessful
         log.error(f"Failed to fetch commits. Status code: {response.status_code}")
-    # Return the list of commit IDs
     return commit_list
 
 
@@ -117,40 +107,38 @@ def add_comment_to_mr(project_id, merge_request_id, comment):
 
 
 @retry(stop_max_attempt_number=3, wait_fixed=2000)
-def get_mr_comment_info(project_id, mr_iid, ):
-    url = f"{gitlab_server_url}/api/v4/projects/{project_id}/merge_requests/{mr_iid}/notes"
+def get_merge_request_comments(project_id, merge_request_iid):
+    url = f"{gitlab_server_url}/api/v4/projects/{project_id}/merge_requests/{merge_request_iid}/notes"
 
-    # 发送API请求
     headers = {"Private-Token": gitlab_private_token}
     response = requests.get(url, headers=headers)
 
-    comments_info = ""
-    # 解析JSON响应
+    comments_content = ""
     if response.status_code == 200:
         comments = response.json()
         for comment in comments:
-            author = comment['author']['username']
-            comment_text = comment['body']
-            print(f"Author: {author}")
-            print(f"Comment: {comment_text}")
-            comments_info += comment_text
+            author_username = comment['author']['username']
+            comment_body = comment['body']
+            print(f"Author: {author_username}")
+            print(f"Comment: {comment_body}")
+            comments_content += comment_body
 
     else:
         print(f" 获取mr comment 失败， Status code: {response.status_code}")
-    return comments_info
+    return comments_content
 
 
 @retry(stop_max_attempt_number=3, wait_fixed=2000)
 def get_commit_change_file(push_info):
     # 获取提交列表
-    commits = push_info['commits']
-    add_file = []
-    modify_file = []
+    commit_list = push_info['commits']
+    added_files_list = []
+    modified_files_list = []
     # 遍历提交
-    for commit in commits:
+    for commit in commit_list:
         added_files = commit.get('added', [])
         modified_files = commit.get('modified', [])
-        add_file += added_files
-        modify_file += modified_files
+        added_files_list += added_files
+        modified_files_list += modified_files
 
-    return add_file + modify_file
+    return added_files_list + modified_files_list

app/gitlab_webhook.py

@@ -2,7 +2,7 @@
 import threading
 from os import abort
 from flask import Blueprint, request, jsonify
-from config.config import WEBHOOK_VERIFY_TOKEN
+# from config.config import WEBHOOK_VERIFY_TOKEN
 from service.chat_review import review_code, review_code_for_mr, review_code_for_add_commit
 from utils.logger import log
 from app.gitlab_utils import get_commit_list, get_merge_request_id, get_commit_change_file
@@ -20,54 +20,54 @@ def question():
 def webhook():
     if request.method == 'GET':
         # 获取gitlab的webhook的token
-        verify_token = request.headers.get('X-Gitlab-Token')
+        webhook_token = request.headers.get('X-Gitlab-Token')
 
         # gitlab的webhook的token验证
-        if verify_token == WEBHOOK_VERIFY_TOKEN:
-            return jsonify({'status': 'success'}), 200
-        else:
-            return jsonify({'status': 'bad token'}), 401
+        # if webhook_token == WEBHOOK_VERIFY_TOKEN:
+        return jsonify({'status': 'success'}), 200
+        #else:
+        #    return jsonify({'status': 'bad token'}), 401
 
     elif request.method == 'POST':
         """
         webhook的主要逻辑,获取gitlab的推送信息
         """
         # 获取gitlab的推送信息
-        gitlab_message = request.data.decode('utf-8')
+        gitlab_payload = request.data.decode('utf-8')
         # 将gitlab的推送信息转换为字典
-        gitlab_message = json.loads(gitlab_message)
-        log.info(f"🌈 ：{gitlab_message}")
+        gitlab_payload = json.loads(gitlab_payload)
+        log.info(f"🌈 ：{gitlab_payload}")
         # 获取项目的类型
-        object_kind = gitlab_message.get('object_kind')
+        event_type = gitlab_payload.get('object_kind')
 
         # 首次发起mr时候触发
-        if object_kind == 'merge_request' and gitlab_message.get("object_attributes").get(
-            "state") == "opened" and gitlab_message.get("object_attributes").get("merge_status") == "preparing":
+        if event_type == 'merge_request' and gitlab_payload.get("object_attributes").get(
+            "state") == "opened" and gitlab_payload.get("object_attributes").get("merge_status") == "preparing":
             # 验证通过，获取commit的信息
-            log.info("首次merge_request ", gitlab_message)
+            log.info("首次merge_request ", gitlab_payload)
             # 获取项目id
-            project_id = gitlab_message.get('project')['id']
+            project_id = gitlab_payload.get('project')['id']
             # 获取merge request ID
-            merge_id = gitlab_message.get("object_attributes")["iid"]
+            merge_request_id = gitlab_payload.get("object_attributes")["iid"]
 
-            thread = threading.Thread(target=review_code_for_mr, args=(project_id, merge_id, gitlab_message))
+            thread = threading.Thread(target=review_code_for_mr, args=(project_id, merge_request_id, gitlab_payload))
             thread.start()
 
             return jsonify({'status': 'success'}), 200
-        elif object_kind == 'push':
+        elif event_type == 'push':
             # 获取merge request ID
-            merge_id = get_merge_request_id(gitlab_message.get('ref').split("/")[-1], gitlab_message.get("project_id"))
+            merge_request_id = get_merge_request_id(gitlab_payload.get('ref').split("/")[-1], gitlab_payload.get("project_id"))
             # 获取项目id
-            project_id = gitlab_message.get('project')['id']
-            if not merge_id:
+            project_id = gitlab_payload.get('project')['id']
+            if not merge_request_id:
                 send_dingtalk_message_by_sign(
-                    f"Project_Name:{gitlab_message['project']['name']}\n备注：分支 {gitlab_message.get('ref')} 没有处于open状态的 Merge Request 不进行 Code Review。")
-                return jsonify({'status': f'非存在MR分支,{gitlab_message}'}), 200
+                    f"Project_Name:{gitlab_payload['project']['name']}\n备注：分支 {gitlab_payload.get('ref')} 没有处于open状态的 Merge Request 不进行 Code Review。")
+                return jsonify({'status': f'非存在MR分支,{gitlab_payload}'}), 200
 
-            change_files = get_commit_change_file(gitlab_message)
+            changed_files = get_commit_change_file(gitlab_payload)
 
             thread = threading.Thread(target=review_code_for_add_commit,
-                                      args=(project_id, merge_id, change_files, gitlab_message))
+                                      args=(project_id, merge_request_id, changed_files, gitlab_payload))
             thread.start()
 
             return jsonify({'status': 'success'}), 200
@@ -76,7 +76,7 @@ def webhook():
             log.error("不是merge")
             return jsonify({'status': '操作不为push'}), 200
 
-        return jsonify({'status': f'未匹配到规则,{gitlab_message}'}), 200
+        return jsonify({'status': f'未匹配到规则,{gitlab_payload}'}), 200
 
     else:
         abort(400)

config/config.py

@@ -2,16 +2,20 @@
 # model list
 model_quester_anster = "text-davinci-003"
 model_gpt_35_turbo = "gpt-3.5-turbo"
+model_gpt_4o = "gpt-4o"
 model_programming_translate = "code-davinci-002"
 
-# gpt key
-openai_api_key = "your openai key"
+# api 接口封装类
+llm_api_impl = "llm_api.llm_api_default.LLMApiDefault"
 
-# openai api
-openai_baseurl = "https://api.openai.com/v1"
-
-# gpt model
-openai_model_name = model_gpt_35_turbo
+# api 配置默使用认UnionLLM，参考：https://github.com/EvalsOne/UnionLLM/tree/main/docs
+# UnionLLM兼容LiteLLM，参考LiteLLM文档：https://docs.litellm.ai/docs
+api_config = {
+    "OPENAI_API_KEY": "your openai key",
+    "OPENAI_API_BASE": "https://api.openai.com/v1",
+    "MODEL_NAME": model_gpt_4o,
+    "PROVIDER": "openai",
+}
 
 # Prompt
 gpt_message = """
@@ -39,7 +43,7 @@
 
 # ------------------Gitlab info--------------------------
 # Gitlab url
-gitlab_server_url = "https://gitlab.example.com"
+gitlab_server_url = "https://gitlab.com"
 
 # Gitlab private token
 gitlab_private_token = "gitlab private token"

doc/config.md

@@ -0,0 +1,22 @@
+# Config.py 配置指南
+## 大模型配置
+- `llm_api_impl`：大模型API接口实现
+  - 默认`llm_api_default`使用`UnionLLM`进行多模型支持，`UnionLLM`兼容`LiteLLM`，
+  支持模型和参数配置方式参见：[UnionLLM仓库](https://github.com/EvalsOne/UnionLLM/)和[LiteLLM文档](https://docs.litellm.ai/docs)。
+  - 主流模型只需要修改`api_config`即可接入，无需修改该参数
+  - 实现默认不支持的大模型接入，可实现`llm_api_interface`接口，并将类名传入该参数。
+- `api_config`: 大模型API配置
+  - 具体配置参见[UnionLLM仓库](https://github.com/EvalsOne/UnionLLM/)和[LiteLLM文档](https://docs.litellm.ai/docs)
+  - `MODEL_NAME` 传入模型名称
+  - `PROVIDER` 传入模型提供商
+  - 该配置会自动传给`llm_api_impl`的`set_config`方法，用于初始化大模型API。 
+
+
+## Gitlab配置
+- `gitlab_server_url`: Gitlab服务器地址
+- `gitlab_private_token`: Gitlab私有令牌
+- `maximum_files`: Gitlab Merge Request最大文件数
+
+## 消息通知配置
+- `dingtalk_webhook`: 钉钉机器人Webhook
+- `dingtalk_secret`: 钉钉机器人密钥