File tree Expand file tree Collapse file tree 1 file changed +9
-4
lines changed Expand file tree Collapse file tree 1 file changed +9
-4
lines changed Original file line number Diff line number Diff line change 1111from contextlib import ExitStack , contextmanager
1212from inspect import isclass , signature
1313from logging import DEBUG
14- from typing import Any , TypeVar
14+ from typing import Any , TypeVar , cast
1515
1616import msgspec
1717import zmq
@@ -382,12 +382,17 @@ def step_with_batch_queue(
382382 deferred_scheduler_output = None
383383 if self .scheduler .has_requests ():
384384 scheduler_output = self .scheduler .schedule ()
385- future = self .model_executor .execute_model (scheduler_output , non_block = True )
385+ exec_future = self .model_executor .execute_model (
386+ scheduler_output , non_block = True
387+ )
386388 if not self .ec_producer :
387389 model_executed = scheduler_output .total_num_scheduled_tokens > 0
388390
389- if model_executed :
390- future .add_done_callback (self ._log_err_callback (scheduler_output ))
391+ if not model_executed :
392+ # No sampling required (no requests scheduled).
393+ future = cast (Future [ModelRunnerOutput ], exec_future )
394+ else :
395+ exec_future .add_done_callback (self ._log_err_callback (scheduler_output ))
391396
392397 if not scheduler_output .pending_structured_output_tokens :
393398 # We aren't waiting for any tokens, get any grammar output
You can’t perform that action at this time.
0 commit comments