We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 56dd0a0 commit 41819d7Copy full SHA for 41819d7
backends/trtllm/src/looper.rs
@@ -98,12 +98,17 @@ fn executor_status_looper(
98
let generation_params = &request.parameters;
99
let stopping_params = &request.stopping_parameters;
100
let input_ids = request.input_ids.as_deref();
101
+ let top_k = if generation_params.do_sample {
102
+ generation_params.top_k
103
+ } else {
104
+ 1
105
+ };
106
107
// Submit to the TensorRT-LLM executor for scheduling
108
match backend.pin_mut().submit(
109
&input_ids.unwrap(), // This is checked beforehand in validate()
110
stopping_params.max_new_tokens,
- generation_params.top_k,
111
+ top_k,
112
generation_params.top_p,
113
generation_params.temperature,
114
generation_params.repetition_penalty,
0 commit comments