Skip to content

Commit 9a5d228

Browse files
ahuang11ahuang11
committed
Fix refresh and also refresh settings
1 parent 4d929a5 commit 9a5d228

File tree

2 files changed

+90
-23
lines changed

2 files changed

+90
-23
lines changed

‎src/panel_web_llm/main.py‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ class WebLLM(JSComponent):
5353
)
5454

5555
system = param.String(
56-
default="Be the most you can be; spread positivity!",
56+
default="Be the best helper!",
5757
doc="The system prompt for the model completion.",
5858
)
5959

@@ -74,6 +74,13 @@ class WebLLM(JSComponent):
7474
Whether the model is currently loading.""",
7575
)
7676

77+
refresh = param.Event(
78+
doc="""
79+
Click to load the latest available models from https://mlc.ai/models.
80+
Requires a network connection.
81+
"""
82+
)
83+
7784
_esm = """
7885
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
7986
@@ -136,6 +143,9 @@ def __init__(self, **params):
136143
self._buffer = []
137144
load_layout = pn.Column if params.get("load_layout") == "column" else pn.Row
138145
self._model_select = pn.widgets.NestedSelect(layout=load_layout)
146+
self._model_select_placeholder = pn.pane.Placeholder(
147+
object=self._model_select,
148+
)
139149
super().__init__(**params)
140150

141151
self._history_input = pn.widgets.IntSlider.from_param(
@@ -148,6 +158,16 @@ def __init__(self, **params):
148158
disabled=self.param.loading,
149159
sizing_mode="stretch_width",
150160
)
161+
self._refresh_button = pn.widgets.ButtonIcon.from_param(
162+
self.param.refresh,
163+
name="",
164+
align="end",
165+
icon="refresh",
166+
active_icon="check",
167+
toggle_duration=1000,
168+
margin=(10, -5, 5, 0),
169+
size="28px",
170+
)
151171
self._load_button = pn.widgets.Button.from_param(
152172
self.param.load_model,
153173
name=param.rx("Load ") + self.param.model_slug,
@@ -158,7 +178,8 @@ def __init__(self, **params):
158178
)
159179
load_status = self.param.load_status.rx()
160180
load_row = load_layout(
161-
self._model_select,
181+
self._model_select_placeholder,
182+
self._refresh_button,
162183
self._load_button,
163184
sizing_mode="stretch_width",
164185
margin=0,
@@ -234,11 +255,19 @@ def _update_model_select(self):
234255
if self.model_slug:
235256
model_params = ModelParam.from_model_slug(self.model_slug)
236257
value = model_params.to_dict(levels)
237-
self._model_select.param.update(
258+
# TODO: Bug https://github.com/holoviz/panel/issues/7647
259+
# self._model_select.param.update(
260+
# options=options,
261+
# levels=levels,
262+
# value=value,
263+
# )
264+
self._model_select = pn.widgets.NestedSelect(
238265
options=options,
239266
levels=levels,
240267
value=value,
268+
layout=self._model_select.layout,
241269
)
270+
self._model_select_placeholder.object = self._model_select
242271
self.param["model_slug"].objects = sorted(value for models in MODEL_MAPPING.values() for sizes in models.values() for value in sizes.values())
243272

244273
def _update_model_slug(self, event):
@@ -333,6 +362,7 @@ async def create_completion(self, messages):
333362
elif reason:
334363
return
335364

365+
@param.depends("refresh", watch=True)
336366
def refresh_model_mapping(self):
337367
"""
338368
Refreshes the model mapping by fetching the latest from the mlc.ai website.
@@ -345,14 +375,18 @@ def refresh_model_mapping(self):
345375
import bs4
346376
import requests # type: ignore
347377

348-
text = requests.get("https://mlc.ai/models#mlc-models").text
378+
try:
379+
text = requests.get("https://mlc.ai/models#mlc-models").text
380+
except requests.ConnectionError:
381+
self._refresh_button.param.update(icon="wifi-off", active_icon="x", description="Connection unavailable.")
382+
return
349383
soup = bs4.BeautifulSoup(text, "html.parser")
350384
table = soup.find("table")
351385
links = table.find_all("a")
352386
model_mapping: dict = {}
353387
for link in links:
354388
model_slug = link.get("href").rsplit("/", 1)[-1]
355-
model_params = ModelParam.from_slug(model_slug)
389+
model_params = ModelParam.from_model_slug(model_slug)
356390
model_name = model_params.model
357391
model_parameters = model_params.size
358392
model_quantization = model_params.quantization

‎src/panel_web_llm/settings.py‎

Lines changed: 51 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,13 @@
7070
"q4f32_1": "Hermes-3-Llama-3.1-8B-q4f32_1-MLC",
7171
}
7272
},
73+
"Hermes-3-Llama-3.2": {
74+
"3B": {
75+
"q0f16": "Hermes-3-Llama-3.2-3B-q0f16-MLC",
76+
"q4f16_1": "Hermes-3-Llama-3.2-3B-q4f16_1-MLC",
77+
"q4f32_1": "Hermes-3-Llama-3.2-3B-q4f32_1-MLC",
78+
}
79+
},
7380
"Phi-3-mini": {
7481
"128k": {
7582
"q0f16": "Phi-3-mini-128k-instruct-q0f16-MLC",
@@ -240,6 +247,40 @@
240247
"q4f32_1": "Qwen2.5-Math-72B-Instruct-q4f32_1-MLC",
241248
},
242249
},
250+
"DeepSeek-R1-Distill-Llama": {
251+
"70B": {
252+
"q0f16": "DeepSeek-R1-Distill-Llama-70B-q0f16-MLC",
253+
"q4f16_1": "DeepSeek-R1-Distill-Llama-70B-q4f16_1-MLC",
254+
"q4f32_1": "DeepSeek-R1-Distill-Llama-70B-q4f32_1-MLC",
255+
},
256+
"8B": {
257+
"q0f16": "DeepSeek-R1-Distill-Llama-8B-q0f16-MLC",
258+
"q4f16_1": "DeepSeek-R1-Distill-Llama-8B-q4f16_1-MLC",
259+
"q4f32_1": "DeepSeek-R1-Distill-Llama-8B-q4f32_1-MLC",
260+
},
261+
},
262+
"DeepSeek-R1-Distill-Qwen": {
263+
"1.5B": {
264+
"q0f16": "DeepSeek-R1-Distill-Qwen-1.5B-q0f16-MLC",
265+
"q4f16_1": "DeepSeek-R1-Distill-Qwen-1.5B-q4f16_1-MLC",
266+
"q4f32_1": "DeepSeek-R1-Distill-Qwen-1.5B-q4f32_1-MLC",
267+
},
268+
"14B": {
269+
"q0f16": "DeepSeek-R1-Distill-Qwen-14B-q0f16-MLC",
270+
"q4f16_1": "DeepSeek-R1-Distill-Qwen-14B-q4f16_1-MLC",
271+
"q4f32_1": "DeepSeek-R1-Distill-Qwen-14B-q4f32_1-MLC",
272+
},
273+
"32B": {
274+
"q0f16": "DeepSeek-R1-Distill-Qwen-32B-q0f16-MLC",
275+
"q4f16_1": "DeepSeek-R1-Distill-Qwen-32B-q4f16_1-MLC",
276+
"q4f32_1": "DeepSeek-R1-Distill-Qwen-32B-q4f32_1-MLC",
277+
},
278+
"7B": {
279+
"q0f16": "DeepSeek-R1-Distill-Qwen-7B-q0f16-MLC",
280+
"q4f16_1": "DeepSeek-R1-Distill-Qwen-7B-q4f16_1-MLC",
281+
"q4f32_1": "DeepSeek-R1-Distill-Qwen-7B-q4f32_1-MLC",
282+
},
283+
},
243284
"DeepSeek-V2-Lite-Chat": {
244285
"-": {
245286
"q0f16": "DeepSeek-V2-Lite-Chat-q0f16-MLC",
@@ -260,46 +301,38 @@
260301
"q0f32": "SmolLM-1.7B-Instruct-q0f32-MLC",
261302
"q4f16_1": "SmolLM-1.7B-Instruct-q4f16_1-MLC",
262303
"q4f32_1": "SmolLM-1.7B-Instruct-q4f32_1-MLC",
263-
}
264-
},
265-
"SmolLM-135M-Instruct": {
266-
"-": {
304+
},
305+
"135M": {
267306
"q0f16": "SmolLM-135M-Instruct-q0f16-MLC",
268307
"q0f32": "SmolLM-135M-Instruct-q0f32-MLC",
269308
"q4f16_1": "SmolLM-135M-Instruct-q4f16_1-MLC",
270309
"q4f32_1": "SmolLM-135M-Instruct-q4f32_1-MLC",
271-
}
272-
},
273-
"SmolLM-360M-Instruct": {
274-
"-": {
310+
},
311+
"360M": {
275312
"q0f16": "SmolLM-360M-Instruct-q0f16-MLC",
276313
"q0f32": "SmolLM-360M-Instruct-q0f32-MLC",
277314
"q4f16_1": "SmolLM-360M-Instruct-q4f16_1-MLC",
278315
"q4f32_1": "SmolLM-360M-Instruct-q4f32_1-MLC",
279-
}
316+
},
280317
},
281318
"SmolLM2": {
282319
"1.7B": {
283320
"q0f16": "SmolLM2-1.7B-Instruct-q0f16-MLC",
284321
"q4f16_1": "SmolLM2-1.7B-Instruct-q4f16_1-MLC",
285322
"q4f32_1": "SmolLM2-1.7B-Instruct-q4f32_1-MLC",
286-
}
287-
},
288-
"SmolLM2-135M-Instruct": {
289-
"-": {
323+
},
324+
"135M": {
290325
"q0f16": "SmolLM2-135M-Instruct-q0f16-MLC",
291326
"q0f32": "SmolLM2-135M-Instruct-q0f32-MLC",
292327
"q4f16_1": "SmolLM2-135M-Instruct-q4f16_1-MLC",
293328
"q4f32_1": "SmolLM2-135M-Instruct-q4f32_1-MLC",
294-
}
295-
},
296-
"SmolLM2-360M-Instruct": {
297-
"-": {
329+
},
330+
"360M": {
298331
"q0f16": "SmolLM2-360M-Instruct-q0f16-MLC",
299332
"q0f32": "SmolLM2-360M-Instruct-q0f32-MLC",
300333
"q4f16_1": "SmolLM2-360M-Instruct-q4f16_1-MLC",
301334
"q4f32_1": "SmolLM2-360M-Instruct-q4f32_1-MLC",
302-
}
335+
},
303336
},
304337
"gemma-2": {
305338
"27b": {

0 commit comments

Comments
 (0)