Upload folder using huggingface_hub

chat_template.jinja

@@ -1,25 +1,7 @@
-{%- for message in messages -%}
-	{%- if loop.first and message["role"] != "system" -%}
-		{{- "<|im_start|>system\nYou are a helpful language and speech assistant.<|im_end|>\n" -}}
-	{%- endif -%}
-	{{- "<|im_start|>" -}}
-	{{- message["role"] -}}
-	{{- "\n" -}}
-	{%- if message["content"] is string -%}
-		{{- message["content"] -}}
-	{%- else -%}
-		{%- for content in message["content"] -%}
-			{%- if content["type"] == "text" -%}
-                {{- content["text"] -}}
-			{%- elif content["type"] == "audio" -%}
-				{{- "<|audio_bos|><|AUDIO|><|audio_eos|>" -}}
-			{%- endif -%}
-		{%- endfor -%}
-	{%- endif -%}
-    {%- if not loop.last or loop.last and not continue_final_message -%}
-        {{- "<|im_end|>\n" -}}
-    {%- endif -%}
-{%- endfor -%}
-{%- if add_generation_prompt -%}
-	{{- "<|im_start|>assistant\n" -}}
-{%- endif -%}
+{% set audio_count = namespace(value=0) %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_bos|><|IMAGE|><|vision_eos|>{% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}{% set audio_count.value = audio_count.value + 1 %}{% if add_audio_id %}Audio {{ audio_count.value }}: {% endif %}<|audio_bos|><|AUDIO|><|audio_eos|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_bos|><|VIDEO|><|vision_eos|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

config.json

@@ -32,6 +32,7 @@
     "target_length": 1008,
     "win_length": 512
   },
+  "audio_token_id": 151646,
   "auto_map": {
     "AutoConfig": "configuration_midashenglm.MiDashengLMConfig",
     "AutoModelForCausalLM": "modeling_midashenglm.MiDashengLMModel"
@@ -63,7 +64,6 @@
     },
     "rope_theta": 1000000.0,
     "sliding_window": 32768,
-    "torch_dtype": "bfloat16",
     "use_cache": true,
     "use_sliding_window": false,
     "vocab_size": 151936

configuration_midashenglm.py

@@ -66,6 +66,7 @@ class MiDashengLMConfig(PretrainedConfig):
         audio_encoder_config: Dict = {},
         subsample_factor: int = 5,
         text_config: Dict = {},
+        audio_token_id: Optional[int] = None,
         **kwargs,
     ):
         self.audio_encoder_config = DashengConfig(**audio_encoder_config)
@@ -75,4 +76,5 @@ class MiDashengLMConfig(PretrainedConfig):
             if text_config
             else Qwen2_5OmniTextConfig()
         )
+        self.audio_token_id = audio_token_id
         super().__init__(**kwargs)

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac83714f7a786cfe80cd40b86b64dc63063f8dbebc34c80298be63218c455ee
+size 4978372408

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:084430974214152e9658155dd21babb35413468bc9025a30820a723c0824ad28
+size 4932950784

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9c20b898e857e682e490a80a602e4b61e79ec2db35ad19ba4cf5720c43301c
+size 4932950856

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e44f1858a81ee7a8dd96cfad57cb0567ed2a5513f0a7d6344b0975579e62b17
+size 1334862432

modeling_midashenglm.py

@@ -1,7 +1,18 @@
 import collections
 import collections.abc
 from dataclasses import dataclass
-from typing import Any, Callable, Iterable, Optional, Sequence, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    Unpack,
+    cast,
+)
 
 import torch
 import torch.nn as nn
@@ -16,6 +27,7 @@ from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
 from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
     Qwen2_5OmniThinkerTextModel,
 )
+from transformers.utils import LossKwargs, can_return_tuple
 
 from .configuration_midashenglm import DashengConfig, MiDashengLMConfig
 
@@ -61,7 +73,7 @@ class AudioPatchEmbed(nn.Module):
         )
         self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
 
-    def forward(self, x):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.proj(x)
         if self.flatten:
             x = torch.permute(
@@ -77,7 +89,7 @@ class LayerScale(nn.Module):
         self.inplace = inplace
         self.gamma = nn.Parameter(init_values * torch.ones(dim))
 
-    def forward(self, x):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         return x.mul_(self.gamma) if self.inplace else x * self.gamma
 
 
@@ -97,7 +109,7 @@ class DashengMlp(nn.Module):
         self.fc2 = nn.Linear(hidden_features, out_features)
         self.drop = nn.Dropout(drop)
 
-    def forward(self, x):
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = self.fc1(x)
         x = self.act(x)
         x = self.drop(x)
@@ -128,7 +140,7 @@ class DashengAttention(nn.Module):
         self.proj_drop = nn.Dropout(proj_drop)
         self.causal = causal
 
-    def forward(self, x, mask: Optional[torch.Tensor] = None):
+    def forward(self, x: torch.Tensor, mask: Optional[torch.Tensor] = None):
         B, N, C = x.shape
         qkv = (
             self.qkv(x)
@@ -206,14 +218,19 @@ class DashengBlock(nn.Module):
         )
 
     # Kwargs usually has a mask parameter that is passed to Attention
-    def forward(self, x, **kwargs):
-        x = x + self.ls1(self.attn(self.norm1(x), **kwargs))
+    def forward(
+        self,
+        x: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        x = x + self.ls1(self.attn(self.norm1(x), mask))
         x = x + self.ls2(self.mlp(self.norm2(x)))
         return x
 
 
 class DashengAudioTransformer(PreTrainedModel):
     config_class = DashengConfig
+    supports_gradient_checkpointing = True
 
     def __init__(self, config: DashengConfig):
         super().__init__(config)
@@ -221,6 +238,7 @@ class DashengAudioTransformer(PreTrainedModel):
         self.target_length = config.target_length
         self.embed_dim = config.embed_dim
         self.hop_length = config.hop_length
+        self.gradient_checkpointing = False
 
         self.front_end = nn.Sequential(
             audio_transforms.MelSpectrogram(
@@ -271,7 +289,11 @@ class DashengAudioTransformer(PreTrainedModel):
 
         self.post_init()
 
-    def forward_features(self, x: torch.Tensor, **kwargs) -> torch.Tensor:
+    def forward_features(
+        self,
+        x: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
         t = x.shape[-1]
         x = x + self.time_pos_embed[:, :, :, :t]
         x = (
@@ -282,7 +304,10 @@ class DashengAudioTransformer(PreTrainedModel):
         )  # rearrange(x, "b c f t -> b (f t) c")
         x = self.pos_drop(x)
         for block in self.blocks:
-            x = block(x, **kwargs)
+            if self.gradient_checkpointing and self.training:
+                x = self._gradient_checkpointing_func(block, x, mask)
+            else:
+                x = block(x, mask)
         x = self.norm(x)
         return x
 
@@ -334,13 +359,19 @@ class DashengAudioTransformer(PreTrainedModel):
 
 
 class AudioProjectorSubsample(nn.Module):
-    def __init__(self, in_dim: int, out_dim: int, downsample_rate=5):
+    def __init__(
+        self,
+        in_dim: int,
+        out_dim: int,
+        downsample_rate=5,
+        dtype: Optional[torch.dtype] = None,
+    ):
         super().__init__()
         self.k = downsample_rate
         self.net = nn.Sequential(
-            nn.Linear(in_dim * self.k, out_dim),
+            nn.Linear(in_dim * self.k, out_dim, dtype=dtype),
             nn.GELU(),
-            nn.Linear(out_dim, out_dim),
+            nn.Linear(out_dim, out_dim, dtype=dtype),
         )
 
     def forward(self, x, mask=None):
@@ -365,6 +396,7 @@ class AudioProjectorSubsample(nn.Module):
 
 @dataclass
 class Qwen25OmniTextModelOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
     logits: Optional[torch.FloatTensor] = None
     past_key_values: Optional[Cache] = None
     hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
@@ -390,12 +422,20 @@ class Qwen25OmniThinkerTextOnlyDecoder(PreTrainedModel, GenerationMixin):
         )
         self.post_init()
 
+    @can_return_tuple
     def forward(
         self,
-        attention_mask: Optional[Tensor] = None,
-        position_ids: Optional[torch.Tensor] = None,
-        return_dict: Optional[bool] = None,
-        **kwargs: Any,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        **kwargs: Unpack[LossKwargs],
     ) -> Union[Tuple, Qwen25OmniTextModelOutput]:
         if attention_mask is not None and position_ids is None:
             position_ids = (
@@ -406,28 +446,33 @@ class Qwen25OmniThinkerTextOnlyDecoder(PreTrainedModel, GenerationMixin):
             )
 
         outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
             attention_mask=attention_mask,
             position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            cache_position=cache_position,
             return_dict=True,
-            **kwargs,
         )
         hidden_states = outputs.last_hidden_state
         logits = self.lm_head(hidden_states)
 
-        if not return_dict:
-            return tuple(
-                v
-                for v in [
-                    logits,
-                    outputs.last_hidden_state,
-                    outputs.past_key_values,
-                    outputs.hidden_states,
-                    outputs.attentions,
-                ]
-                if v is not None
+        loss = (
+            self.loss_function(
+                logits=logits,
+                labels=labels,
+                vocab_size=self.config.vocab_size,
+                **kwargs,
             )
+            if labels is not None
+            else None
+        )
 
         return Qwen25OmniTextModelOutput(
+            loss=loss,
             logits=logits,
             past_key_values=outputs.past_key_values,
             hidden_states=outputs.hidden_states,
@@ -443,12 +488,17 @@ class MiDashengLMModel(PreTrainedModel):
     _supports_cache_class = Qwen2_5OmniThinkerTextModel._supports_cache_class
     _supports_static_cache = Qwen2_5OmniThinkerTextModel._supports_static_cache
     _supports_quantized_cache = Qwen2_5OmniThinkerTextModel._supports_quantized_cache
+    supports_gradient_checkpointing = (
+        Qwen2_5OmniThinkerTextModel.supports_gradient_checkpointing
+    )
 
     def __init__(self, config: MiDashengLMConfig):
         super().__init__(config)
 
+        self.audio_token_id = config.audio_token_id
+
         self.audio_encoder = DashengAudioTransformer._from_config(
-            config.audio_encoder_config
+            config.audio_encoder_config,
         )
         self.audio_projector = AudioProjectorSubsample(
             self.audio_encoder.embed_dim,
@@ -480,7 +530,6 @@ class MiDashengLMModel(PreTrainedModel):
         input_values: Optional[torch.Tensor],
         inputs_embeds: Optional[torch.Tensor],
         audio_length: Optional[Iterable[int]] = None,
-        audio_token_id: Optional[int] = None,
     ) -> torch.Tensor:
         if input_ids is not None:
             if inputs_embeds is not None:
@@ -492,9 +541,9 @@ class MiDashengLMModel(PreTrainedModel):
             )
 
             if input_values is not None:
-                if audio_token_id is None:
+                if self.audio_token_id is None:
                     raise ValueError(
-                        "If `input_values` is provided, `audio_token_id` must also be provided."
+                        "Audio input is provided, but `audio_token_id` is not configured."
                     )
 
                 audio_embeddings = self._forward_audio_encoder(
@@ -502,7 +551,7 @@ class MiDashengLMModel(PreTrainedModel):
                     audio_length=audio_length,
                 ).to(inputs_embeds.dtype)
 
-                audio_mask = (input_ids == audio_token_id).flatten()
+                audio_mask = (input_ids == self.audio_token_id).flatten()
                 diff = torch.diff(
                     audio_mask.long(),
                     prepend=torch.zeros(
@@ -540,7 +589,9 @@ class MiDashengLMModel(PreTrainedModel):
         input_values: Optional[Tensor] = None,
         inputs_embeds: Optional[Tensor] = None,
         audio_length: Optional[Iterable[int]] = None,
-        audio_token_id: Optional[int] = None,
+        attention_mask: Optional[Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
         **kwargs: Any,
     ):
         inputs_embeds = self._prepare_inputs_embeds(
@@ -548,11 +599,13 @@ class MiDashengLMModel(PreTrainedModel):
             input_values=input_values,
             inputs_embeds=inputs_embeds,
             audio_length=audio_length,
-            audio_token_id=audio_token_id,
         )
         return self.decoder(
             input_ids=None,
             inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            labels=labels,
             **kwargs,
         )
 
@@ -562,7 +615,6 @@ class MiDashengLMModel(PreTrainedModel):
         input_values: Optional[Tensor] = None,
         inputs_embeds: Optional[Tensor] = None,
         audio_length: Optional[Iterable[int]] = None,
-        audio_token_id: Optional[int] = None,
         **kwargs,
     ):
         inputs_embeds = self._prepare_inputs_embeds(
@@ -570,7 +622,6 @@ class MiDashengLMModel(PreTrainedModel):
             input_values=input_values,
             inputs_embeds=inputs_embeds,
             audio_length=audio_length,
-            audio_token_id=audio_token_id,
         )
         return self.decoder.generate(
             inputs_embeds=inputs_embeds,

processing.py

@@ -1,277 +0,0 @@
-from __future__ import annotations
-
-from typing import List
-
-import numpy as np
-import torch
-from transformers import Qwen2Tokenizer, Qwen2TokenizerFast, Wav2Vec2FeatureExtractor
-from transformers.feature_extraction_utils import BatchFeature
-from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
-
-
-class MiAudioLLMProcessorKwargs(ProcessingKwargs):
-    _defaults = {
-        "text_kwargs": {
-            "padding": True,
-            "padding_side": "left",
-        },
-        "audio_kwargs": {},
-    }
-
-
-def calculate_mel_frames_dasheng(
-    audio_length_samples: int,
-    n_fft: int = 512,
-    hop_size: int = 160,
-    dasheng_subsampling: int = 4,
-    center=True,
-    model_subsampling: int = 5,
-) -> int:
-    """Calculate the number of Mel-spectrogram frames."""
-    if center:
-        audio_length_samples = audio_length_samples + n_fft
-
-    return (
-        int(1 + ((audio_length_samples - n_fft) / hop_size))
-        // dasheng_subsampling
-        // model_subsampling
-    )
-
-
-class MiAudioLLMProcessor(ProcessorMixin):
-    attributes = ["feature_extractor", "tokenizer"]
-    valid_kwargs = [
-        "chat_template",
-        "audio_token",
-        "audio_bos_token",
-        "audio_eos_token",
-    ]
-    feature_extractor_class = "Wav2Vec2FeatureExtractor"
-    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
-
-    def __init__(
-        self,
-        feature_extractor: Wav2Vec2FeatureExtractor | None = None,
-        tokenizer: Qwen2Tokenizer | Qwen2TokenizerFast | None = None,
-        model_subsampling: int = 5,
-        chat_template: str | None = None,
-        # TODO 是否可以移除？
-        audio_token: str = "<|AUDIO|>",
-        audio_bos_token: str = "<|audio_bos|>",
-        audio_eos_token: str = "<|audio_eos|>",
-    ):
-        if chat_template is None:
-            chat_template = self.default_chat_template
-        assert tokenizer is not None, "Tokenizer Needs to be passed"
-        self.audio_token = (
-            tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token
-        )
-        self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token)
-        self.audio_bos_token = (
-            tokenizer.audio_bos_token
-            if hasattr(tokenizer, "audio_bos_token")
-            else audio_bos_token
-        )
-        self.audio_eos_token = (
-            tokenizer.audio_eos_token
-            if hasattr(tokenizer, "audio_eos_token")
-            else audio_eos_token
-        )
-        self.model_subsampling = model_subsampling
-        # Fix Normalization
-        if feature_extractor is not None and feature_extractor.do_normalize is True:
-            feature_extractor.do_normalize = False
-        super().__init__(feature_extractor, tokenizer, chat_template=chat_template)
-
-    def __call__(
-        self,
-        text: List[str] | None = None,
-        audio: List[np.ndarray] | List[torch.Tensor] | None = None,
-        **kwargs: Unpack[MiAudioLLMProcessorKwargs],
-    ) -> BatchFeature:
-        if text is None:
-            raise ValueError("You need to specify `text` input to process.")
-        elif isinstance(text, str):
-            text = [text]
-        elif not isinstance(text, list) and not isinstance(text[0], str):
-            raise ValueError(
-                "Invalid input text. Please provide a string, or a list of strings"
-            )
-
-        output_kwargs = self._merge_kwargs(
-            MiAudioLLMProcessorKwargs,
-            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
-            **kwargs,
-        )
-
-        if audio is not None:
-            if isinstance(audio[0], torch.Tensor):
-                audio = [sample_.numpy() for sample_ in audio]
-
-            if isinstance(audio[0], torch.Tensor):
-                audio = [sample_.squeeze(0) for sample_ in audio]
-                if not all(x_.ndim == 1 for x_ in audio):
-                    raise ValueError("All samples in a list must be 1D.")
-            if isinstance(audio[0], np.ndarray):
-                if not all(x_.ndim == 1 for x_ in audio):
-                    raise ValueError("All samples in a list must be 1D.")
-            # ensure we have as much audios as audio tokens
-            num_audio_tokens = sum(sample.count(self.audio_token) for sample in text)
-            num_audios = 1 if type(audio) is np.ndarray else len(audio)
-            if num_audio_tokens != num_audios:
-                raise ValueError(
-                    f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}"
-                )
-
-            # Some kwargs should not be changed so we can expand text with audio tokens below
-            output_kwargs["audio_kwargs"]["return_attention_mask"] = True
-            output_kwargs["audio_kwargs"]["padding"] = True
-            output_kwargs["audio_kwargs"]["return_tensors"] = "pt"
-
-            # + Padding
-            audio_inputs = self.feature_extractor(
-                audio, **output_kwargs["audio_kwargs"]
-            )
-
-            # remove attention mask, dasheng uses lengths
-            audio_feature_mask = audio_inputs.pop("attention_mask")
-
-            expanded_text = []
-            audio_lengths = audio_feature_mask.sum(-1).tolist()
-            audio_inputs["audio_length"] = torch.tensor(audio_lengths).long()
-            audio_inputs["audio_token_id"] = (
-                self.audio_token_id
-            )  # Pass to the model such that i knows what is the placeholder id
-
-            for sample in text:
-                replace_str = []
-                while self.audio_token in sample:
-                    audio_length = audio_lengths.pop(0)
-                    num_audio_tokens = calculate_mel_frames_dasheng(
-                        audio_length, model_subsampling=self.model_subsampling
-                    )
-
-                    expanded_audio_token = self.audio_token * num_audio_tokens
-
-                    audio_token_start_idx = sample.find(self.audio_token)
-                    audio_token_end_idx = audio_token_start_idx + len(self.audio_token)
-
-                    has_bos = (
-                        sample[
-                            audio_token_start_idx
-                            - len(self.audio_bos_token) : audio_token_start_idx
-                        ]
-                        == self.audio_bos_token
-                    )
-                    has_eos = (
-                        sample[
-                            audio_token_end_idx : audio_token_end_idx
-                            + len(self.audio_eos_token)
-                        ]
-                        == self.audio_eos_token
-                    )
-
-                    # Check if this audio token is surrounded by bos/eos tokens
-                    if not has_bos and not has_eos:
-                        expanded_audio_token = (
-                            self.audio_bos_token
-                            + expanded_audio_token
-                            + self.audio_eos_token
-                        )
-
-                    replace_str.append(expanded_audio_token)
-                    sample = sample.replace(self.audio_token, "<placeholder>", 1)
-
-                while "<placeholder>" in sample:
-                    sample = sample.replace("<placeholder>", replace_str.pop(0), 1)
-                expanded_text.append(sample)
-            text = expanded_text
-
-        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", "pt")
-        inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
-        if hasattr(self, "_check_special_mm_tokens"):
-            self._check_special_mm_tokens(text, inputs, modalities=["audio"])
-
-        if audio is not None:
-            inputs.update(audio_inputs)
-
-        return BatchFeature(data={**inputs}, tensor_type=return_tensors)
-
-    def batch_decode(self, *args, **kwargs):
-        """
-        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
-        refer to the docstring of this method for more information.
-        """
-        return self.tokenizer.batch_decode(*args, **kwargs)
-
-    def decode(self, *args, **kwargs):
-        """
-        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
-        the docstring of this method for more information.
-        """
-        return self.tokenizer.decode(*args, **kwargs)
-
-    @property
-    def model_input_names(self):
-        tokenizer_input_names = self.tokenizer.model_input_names
-        feature_extractor_input_names = self.feature_extractor.model_input_names
-        return list(
-            dict.fromkeys(
-                tokenizer_input_names + feature_extractor_input_names + ["audio_length"]
-            )
-        )
-
-    @property
-    # NOTE: we don't have default templates anymore, and the below is kept only because the hub config is not yet updated!
-    def default_chat_template(self):
-        """
-        This default vicuna template formats inputs in the form of a chat history. For each message in the chat history:
-        * the template will output the role of the speaker followed by the content of the message.
-        * content is a list of strings and audios.
-        * If the content element is an audio, the template will output a sequence of <|AUDIO|> tokens
-
-        Example:
-
-        ```python
-        messages = [
-            {'role': 'system', 'content': 'You are a helpful assistant.'},
-            {"role": "user", "content": [
-                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/glass-breaking-151256.mp3"},
-                {"type": "text", "text": "What's that sound?"},
-            ]},
-            {"role": "assistant", "content": "It is the sound of glass shattering."},
-            {"role": "user", "content": [
-                {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/f2641_0_throatclearing.wav"},
-                {"type": "text", "text": "How about this one?"},
-            ]},
-        ]
-
-        result = template.render(messages=messages, add_generation_prompt=True)
-        ```
-        """
-        # fmt: off
-        return (
-            "{% set audio_count = namespace(value=0) %}"
-            "{% for message in messages %}"
-                "{% if loop.first and message['role'] != 'system' %}"
-                    "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
-                "{% endif %}"
-                "<|im_start|>{{ message['role'] }}\n"
-                "{% if message['content'] is string %}"
-                    "{{ message['content'] }}<|im_end|>\n"
-                "{% else %}"
-                    "{% for content in message['content'] %}"
-                        "{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}"
-                            "{% set audio_count.value = audio_count.value + 1 %}"
-                            "Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
-                        "{% elif 'text' in content %}"
-                            "{{ content['text'] }}"
-                        "{% endif %}"
-                    "{% endfor %}"
-                    "<|im_end|>\n"
-                "{% endif %}"
-            "{% endfor %}"
-            "{% if add_generation_prompt %}"
-                "<|im_start|>assistant\n"
-            "{% endif %}"
-        )

processing_midashenglm.py

@@ -207,9 +207,6 @@ class MiDashengLMProcessor(ProcessorMixin):
             expanded_text = []
             audio_lengths = audio_feature_mask.sum(-1).tolist()
             audio_inputs["audio_length"] = torch.tensor(audio_lengths).long()
-            audio_inputs["audio_token_id"] = (
-                self.audio_token_id
-            )  # Pass to the model such that i knows what is the placeholder id
 
             for sample in text:
                 replace_str = []