|
| 1 | +# |
| 2 | +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. |
| 3 | +# This file is a part of the vllm-ascend project. |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | +# |
| 17 | + |
| 18 | +import torch |
| 19 | +import torch.nn as nn |
| 20 | +from vllm.model_executor.models.qwen2_5_omni_thinker import ( |
| 21 | + Qwen2_5_VLImageInputs, Qwen2_5_VLVideoInputs, |
| 22 | + Qwen2_5OmniThinkerForConditionalGeneration) |
| 23 | + |
| 24 | +from vllm_ascend.ascend_forward_context import set_ascend_forward_context |
| 25 | + |
| 26 | + |
| 27 | +class AscendQwen2_5OmniThinkerForConditionalGeneration(nn.Module): |
| 28 | + |
| 29 | + def _process_image_input( |
| 30 | + self, |
| 31 | + image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]: |
| 32 | + if image_input["type"] == "image_embeds": |
| 33 | + return image_input["image_embeds"].type(self.visual.dtype) |
| 34 | + |
| 35 | + grid_thw = image_input["image_grid_thw"] |
| 36 | + assert grid_thw.ndim == 2 |
| 37 | + |
| 38 | + pixel_values = image_input["pixel_values"].type(self.visual.dtype) |
| 39 | + with set_ascend_forward_context(None, self.vllm_config): |
| 40 | + image_embeds = self.visual(pixel_values, grid_thw=grid_thw) |
| 41 | + # Split concatenated embeddings for each image item. |
| 42 | + merge_size = self.visual.spatial_merge_size |
| 43 | + sizes = grid_thw.prod(-1) // merge_size // merge_size |
| 44 | + |
| 45 | + return image_embeds.split(sizes.tolist()) |
| 46 | + |
| 47 | + def _process_video_input( |
| 48 | + self, |
| 49 | + video_input: Qwen2_5_VLVideoInputs, |
| 50 | + video_hashes: list[str] | None = None, |
| 51 | + cached_video_embeds: torch.Tensor | None = None, |
| 52 | + ) -> torch.Tensor: |
| 53 | + if video_input["type"] == "video_embeds": |
| 54 | + return video_input["video_embeds"].type(self.visual.dtype) |
| 55 | + |
| 56 | + grid_thw = video_input["video_grid_thw"] |
| 57 | + assert grid_thw.ndim == 2 |
| 58 | + |
| 59 | + pixel_values_videos = video_input["pixel_values_videos"].type( |
| 60 | + self.visual.dtype) |
| 61 | + with set_ascend_forward_context(None, self.vllm_config): |
| 62 | + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw) |
| 63 | + # Split concatenated embeddings for each video item. |
| 64 | + merge_size = self.visual.spatial_merge_size |
| 65 | + sizes = grid_thw.prod(-1) // merge_size // merge_size |
| 66 | + |
| 67 | + return video_embeds.split(sizes.tolist()) |
| 68 | + |
| 69 | + |
| 70 | +# NOTE: These will be removed after https://github.com/vllm-project/vllm/pull/29388 is merged. |
| 71 | +Qwen2_5OmniThinkerForConditionalGeneration._process_image_input = AscendQwen2_5OmniThinkerForConditionalGeneration._process_image_input |
| 72 | +Qwen2_5OmniThinkerForConditionalGeneration._process_video_input = AscendQwen2_5OmniThinkerForConditionalGeneration._process_video_input |
0 commit comments