up

Author: juncaipeng

fastdeploy/demo/offline_demo.py

@@ -17,11 +17,9 @@
 from fastdeploy.engine.sampling_params import SamplingParams
 from fastdeploy.entrypoints.llm import LLM
 
-model_name_or_path = "/workspace/ERNIE-4.5-0.3B-Paddle"
-
-# 超参设置
-sampling_params = SamplingParams(temperature=0.1, max_tokens=30, prompt_logprobs=100)
-llm = LLM(model=model_name_or_path, tensor_parallel_size=1, enable_prefix_caching=False)
-output = llm.generate(prompts="who are you？", use_tqdm=True, sampling_params=sampling_params)
+model_name_or_path = "PaddlePaddle/ERNIE-4.5-0.3B-Paddle"
+sampling_params = SamplingParams(temperature=0.1, max_tokens=30)
+llm = LLM(model=model_name_or_path)
+output = llm.generate(prompts="who are you?", use_tqdm=True, sampling_params=sampling_params)
 
 print(output)

fastdeploy/engine/engine.py

@@ -325,6 +325,7 @@ def add_requests(self, task, sampling_params=None, **kwargs):
                 raise EngineError(err_msg, error_code=400)
 
         request.metrics.preprocess_end_time = time.time()
+        request.metrics.scheduler_recv_req_time = time.time()
         self.engine.scheduler.put_requests([request])
         llm_logger.info(f"Cache task with request_id ({request.get('request_id')})")
         llm_logger.debug(f"cache task: {request}")

fastdeploy/engine/request.py

@@ -523,6 +523,8 @@ def record_recv_token(self, cur_time: float = None):
         cur_time = time.time() if cur_time is None else cur_time
         self.engine_recv_latest_token_time = cur_time
         self.llm_engine_recv_latest_token_timestamp = cur_time
+        self.model_execute_time = cur_time - self.arrival_time
+        self.model_forward_time = cur_time - self.inference_start_time
 
     def record_decode_recv_second_token(self):
         cur_time = time.time()
@@ -540,7 +542,6 @@ def cal_cost_time(self):
         self.first_token_time = self.engine_recv_first_token_time - self.inference_start_time
         self.time_in_queue = time.time() - self.preprocess_end_time
         self.preprocess_cost_time = self.preprocess_end_time - self.preprocess_start_time
-        self.model_execute_time = self.engine_recv_first_token_time - self.inference_start_time
         self.request_start_time = self.arrival_time
 
         # for compatibility with old metrics
@@ -623,10 +624,12 @@ def add(self, next_output: RequestOutput) -> None:
         self.outputs.index = next_output.outputs.index
         self.outputs.token_ids.extend(next_output.outputs.token_ids)
 
-        if next_output.metrics.arrival_time is not None and self.metrics.inference_start_time is not None:
-            self.metrics.model_forward_time = next_output.metrics.arrival_time - self.metrics.inference_start_time
-        if next_output.metrics.arrival_time is not None and self.metrics.arrival_time is not None:
-            self.metrics.model_execute_time = next_output.metrics.arrival_time - self.metrics.arrival_time
+        if next_output.metrics.model_forward_time is not None:
+            self.metrics.model_forward_time = next_output.metrics.model_forward_time
+        if next_output.metrics.model_execute_time is not None:
+            self.metrics.model_execute_time = next_output.metrics.model_execute_time
+        if next_output.metrics.engine_recv_latest_token_time is not None:
+            self.metrics.engine_recv_latest_token_time = next_output.metrics.engine_recv_latest_token_time
         if next_output.outputs.top_logprobs is not None:
             self.outputs.top_logprobs.logprob_token_ids.extend(next_output.outputs.top_logprobs.logprob_token_ids)
             self.outputs.top_logprobs.logprobs.extend(next_output.outputs.top_logprobs.logprobs)