made it run

Author: Yuval-Roth

.gitignore

@@ -142,7 +142,7 @@ dmypy.json
 **/multicache_serving.py
 **/modelcache_serving.py
 
-**/model/
+**/model/text2vec-base-chinese
 
 /data/milvus/db
 /data/mysql/db

docker-compose.yaml

@@ -1,7 +1,7 @@
-version: 'Beta'
+name: "modelcache"
 services:
   mysql:
-    image: mysql:8.0.23
+    image: mysql:9.3.0
     container_name: mysql
     environment:
       MYSQL_ROOT_PASSWORD: 'root'
@@ -19,7 +19,7 @@ services:
       - modelcache
 
   milvus:
-    image: milvusdb/milvus:v2.5.0-beta
+    image: milvusdb/milvus:v2.5.10
     container_name: milvus
     security_opt:
       - seccomp:unconfined
@@ -47,24 +47,24 @@ services:
     restart: on-failure  
     command: milvus run standalone
 
-  modelcache:
-    build: 
-      context: .
-      dockerfile: Dockerfile
-    container_name: modelcache
-    image: modelcache:0.1.0
-    ports:
-      - 5000:5000
-    volumes:
-      - ./model:/home/user/model
-      - ./modelcache:/home/user/modelcache
-      - ./modelcache_mm:/home/user/modelcache_mm
-      - ./fastapi4modelcache.py:/home/user/fastapi4modelcache.py
-    networks:
-      - modelcache
-    restart: on-failure
-    command: sh -c "uvicorn fastapi4modelcache:app --reload --reload-dir /home/user --port=5000 --host=0.0.0.0"
+#  modelcache:
+#    build:
+#      context: .
+#      dockerfile: Dockerfile
+#    container_name: modelcache
+#    image: modelcache:0.1.0
+#    ports:
+#      - 5000:5000
+#    volumes:
+#      - ./model:/home/user/model
+#      - ./modelcache:/home/user/modelcache
+#      - ./modelcache_mm:/home/user/modelcache_mm
+#      - ./fastapi4modelcache.py:/home/user/fastapi4modelcache.py
+#    networks:
+#      - modelcache
+#    restart: on-failure
+#    command: sh -c "uvicorn fastapi4modelcache:app --reload --reload-dir /home/user --port=5000 --host=0.0.0.0"
 
 networks:
   modelcache: 
-    external: true
+    driver: bridge

flask4modelcache.py

@@ -38,16 +38,16 @@ def response_hitquery(cache_resp):
 milvus_config = configparser.ConfigParser()
 milvus_config.read('modelcache/config/milvus_config.ini')
 
-es_config = configparser.ConfigParser()
-es_config.read('modelcache/config/elasticsearch_config.ini')
+# es_config = configparser.ConfigParser()
+# es_config.read('modelcache/config/elasticsearch_config.ini')
 
 # redis_config = configparser.ConfigParser()
 # redis_config.read('modelcache/config/redis_config.ini')
 
 # chromadb_config = configparser.ConfigParser()
 # chromadb_config.read('modelcache/config/chromadb_config.ini')
 
-data_manager = get_data_manager(CacheBase("elasticsearch", config=es_config),
+data_manager = get_data_manager(CacheBase("mysql", config=mysql_config),
                                 VectorBase("milvus", dimension=data2vec.dimension, milvus_config=milvus_config))
 
 
@@ -77,18 +77,19 @@ def first_flask():  # 视图函数
 
 @app.route('/modelcache', methods=['GET', 'POST'])
 def user_backend():
+    param_dict = []
     try:
         if request.method == 'POST':
-            request_data = request.json
+            param_dict = request.json
         elif request.method == 'GET':
-            request_data = request.args
-        param_dict = json.loads(request_data)
+            param_dict = request.args
     except Exception as e:
         result = {"errorCode": 101, "errorDesc": str(e), "cacheHit": False, "delta_time": 0, "hit_query": '',
                   "answer": ''}
         cache.data_manager.save_query_resp(result, model='', query='', delta_time=0)
         return json.dumps(result)
 
+
     # param parsing
     try:
         request_type = param_dict.get("type")
@@ -197,4 +198,4 @@ def user_backend():
 
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=True)
+    app.run(host='0.0.0.0', port=5000)

model/clone_model_repository.bat

@@ -0,0 +1,2 @@
+git lfs install
+git clone https://huggingface.co/shibing624/text2vec-base-chinese

modelcache/adapter/adapter_query.py

@@ -7,7 +7,7 @@
 from modelcache.processor.pre import multi_analysis
 from FlagEmbedding import FlagReranker
 
-USE_RERANKER = True  # 如果为 True 则启用 reranker，否则使用原有逻辑
+USE_RERANKER = False  # 如果为 True 则启用 reranker，否则使用原有逻辑
 
 def adapt_query(cache_data_convert, *args, **kwargs):
     chat_cache = kwargs.pop("cache_obj", cache)
@@ -74,7 +74,7 @@ def adapt_query(cache_data_convert, *args, **kwargs):
                 extra_param=context.get("evaluation_func", None),
             )
         if rank_pre < rank_threshold:
-            return
+            return None
 
         if USE_RERANKER:
             reranker = FlagReranker('BAAI/bge-reranker-v2-m3', use_fp16=False)
@@ -86,7 +86,7 @@ def adapt_query(cache_data_convert, *args, **kwargs):
                 if ret is None:
                     continue
 
-                rank = reranker.compute_score([pre_embedding_data, ret[0]], normalize=True)
+                rank = reranker.compute_score([pre_embedding_data, ret[0]], normalize=True)[0]
 
                 if "deps" in context and hasattr(ret.question, "deps"):
                     eval_query_data = {
@@ -192,4 +192,6 @@ def adapt_query(cache_data_convert, *args, **kwargs):
                 logging.info('update_hit_count except, please check!')
 
             chat_cache.report.hint_cache()
-            return cache_data_convert(return_message, return_query)
+            return cache_data_convert(return_message, return_query)
+        return None
+    return None

modelcache/config/milvus_config.ini

@@ -1,5 +1,5 @@
 [milvus]
-host = milvus
+host = localhost
 port = 19530
 user = ''
 password = ''

modelcache/config/mysql_config.ini

@@ -1,5 +1,5 @@
 [mysql]
-host = mysql
+host = localhost
 port = 3306
 username = modelcache
 password = modelcache

requirements.txt

@@ -4,16 +4,18 @@ Flask==3.0.0
 numpy==1.24.4
 onnxruntime==1.16.1
 openai==0.28.1
-pymilvus==2.3.1
-PyMySQL==1.1.0
-Requests==2.31.0
-torch==2.1.1
-transformers==4.38.2
+pymilvus==2.5.9
+PyMySQL==1.1.1
+Requests==2.32.3
+torch==2.7.0
+transformers==4.44.2
 faiss-cpu==1.7.4
 redis==5.0.1
-modelscope==1.14.0
+modelscope==1.26.0
 fastapi==0.115.5
 uvicorn==0.32.0
 chromadb==0.5.23
 elasticsearch==7.10.0
 snowflake-id==1.0.2
+flagembedding==1.3.4
+cryptography==45.0.2