Skip to content

Commit c458d21

Browse files
committed
feat(trtllm): add new finish reasons
Add new finish reasons introduced in TensorRT-LLM v0.16.0.
1 parent 58934c8 commit c458d21

File tree

2 files changed

+12
-0
lines changed

2 files changed

+12
-0
lines changed

backends/trtllm/csrc/ffi.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ namespace huggingface::tgi::backends::trtllm {
4242
return finish_reason_t::kEND_ID;
4343
case tle::FinishReason::kLENGTH:
4444
return finish_reason_t::kLENGTH;
45+
case tle::FinishReason::kTIMED_OUT:
46+
return finish_reason_t::kTIMED_OUT;
47+
case tle::FinishReason::kCANCELLED:
48+
return finish_reason_t::kCANCELLED;
4549
default:
4650
std::unreachable();
4751
}

backends/trtllm/src/lib.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ mod ffi {
2424
/// The request finished because the maximum number of tokens was reached.
2525
#[cxx_name = "kLENGTH"]
2626
MaxLength = 3u8,
27+
28+
#[cxx_name = "kTIMED_OUT"]
29+
/// The request finished because it got timed out (via the mAllotedTime parameter)
30+
TimedOut = 4u8,
31+
32+
#[cxx_name = "kCANCELLED"]
33+
/// The request was cancelled by calling cancelRequest.
34+
Cancelled = 5u8,
2735
}
2836

2937
/// Struct used as shared type between rust and C++ to represent the result

0 commit comments

Comments
 (0)