130 lines
5.3 KiB
Python
130 lines
5.3 KiB
Python
# Copyright (c) 2021 Mobvoi Inc. (authors: Binbin Zhang)
|
|
# 2024 Alibaba Inc (authors: Xiang Lyu, Zetao Hu)
|
|
# 2025 Alibaba Inc (authors: Xiang Lyu, Yabin Li)
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import json
|
|
import torch
|
|
import torchaudio
|
|
import logging
|
|
logging.getLogger('matplotlib').setLevel(logging.WARNING)
|
|
logging.basicConfig(level=logging.DEBUG,
|
|
format='%(asctime)s %(levelname)s %(message)s')
|
|
|
|
|
|
def read_lists(list_file):
|
|
lists = []
|
|
with open(list_file, 'r', encoding='utf8') as fin:
|
|
for line in fin:
|
|
lists.append(line.strip())
|
|
return lists
|
|
|
|
|
|
def read_json_lists(list_file):
|
|
lists = read_lists(list_file)
|
|
results = {}
|
|
for fn in lists:
|
|
with open(fn, 'r', encoding='utf8') as fin:
|
|
results.update(json.load(fin))
|
|
return results
|
|
|
|
|
|
def load_wav(wav, target_sr):
|
|
speech, sample_rate = torchaudio.load(wav, backend='soundfile')
|
|
speech = speech.mean(dim=0, keepdim=True)
|
|
if sample_rate != target_sr:
|
|
assert sample_rate > target_sr, 'wav sample rate {} must be greater than {}'.format(sample_rate, target_sr)
|
|
speech = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_sr)(speech)
|
|
return speech
|
|
|
|
|
|
def convert_onnx_to_trt(trt_model, trt_kwargs, onnx_model, fp16):
|
|
import tensorrt as trt
|
|
logging.info("Converting onnx to trt...")
|
|
network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
|
|
logger = trt.Logger(trt.Logger.INFO)
|
|
builder = trt.Builder(logger)
|
|
network = builder.create_network(network_flags)
|
|
parser = trt.OnnxParser(network, logger)
|
|
config = builder.create_builder_config()
|
|
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 32) # 4GB
|
|
if fp16:
|
|
config.set_flag(trt.BuilderFlag.FP16)
|
|
profile = builder.create_optimization_profile()
|
|
# load onnx model
|
|
with open(onnx_model, "rb") as f:
|
|
if not parser.parse(f.read()):
|
|
for error in range(parser.num_errors):
|
|
print(parser.get_error(error))
|
|
raise ValueError('failed to parse {}'.format(onnx_model))
|
|
# set input shapes
|
|
for i in range(len(trt_kwargs['input_names'])):
|
|
profile.set_shape(trt_kwargs['input_names'][i], trt_kwargs['min_shape'][i], trt_kwargs['opt_shape'][i], trt_kwargs['max_shape'][i])
|
|
tensor_dtype = trt.DataType.HALF if fp16 else trt.DataType.FLOAT
|
|
# set input and output data type
|
|
for i in range(network.num_inputs):
|
|
input_tensor = network.get_input(i)
|
|
input_tensor.dtype = tensor_dtype
|
|
for i in range(network.num_outputs):
|
|
output_tensor = network.get_output(i)
|
|
output_tensor.dtype = tensor_dtype
|
|
config.add_optimization_profile(profile)
|
|
engine_bytes = builder.build_serialized_network(network, config)
|
|
# save trt engine
|
|
with open(trt_model, "wb") as f:
|
|
f.write(engine_bytes)
|
|
logging.info("Succesfully convert onnx to trt...")
|
|
|
|
|
|
def export_cosyvoice2_vllm(model, model_path, device):
|
|
if os.path.exists(model_path):
|
|
return
|
|
pad_to = DEFAULT_VOCAB_PADDING_SIZE = 64
|
|
vocab_size = model.speech_embedding.num_embeddings
|
|
feature_size = model.speech_embedding.embedding_dim
|
|
pad_vocab_size = ((vocab_size + pad_to - 1) // pad_to) * pad_to
|
|
|
|
dtype = torch.bfloat16
|
|
# lm_head
|
|
new_lm_head = torch.nn.Linear(in_features=feature_size, out_features=pad_vocab_size, bias=True)
|
|
with torch.no_grad():
|
|
new_lm_head.weight[:vocab_size] = model.llm_decoder.weight
|
|
new_lm_head.bias[:vocab_size] = model.llm_decoder.bias
|
|
new_lm_head.weight[vocab_size:] = 0
|
|
new_lm_head.bias[vocab_size:] = 0
|
|
model.llm.model.lm_head = new_lm_head
|
|
new_codec_embed = torch.nn.Linear(in_features=feature_size, out_features=pad_vocab_size)
|
|
# embed_tokens
|
|
embed_tokens = model.llm.model.model.embed_tokens
|
|
with torch.no_grad():
|
|
new_codec_embed.weight[:vocab_size] = model.speech_embedding.weight
|
|
new_codec_embed.weight[vocab_size:] = 0
|
|
model.llm.model.set_input_embeddings(new_codec_embed)
|
|
model.llm.model.to(device)
|
|
model.llm.model.to(dtype)
|
|
tmp_vocab_size = model.llm.model.config.vocab_size
|
|
tmp_tie_embedding = model.llm.model.config.tie_word_embeddings
|
|
del model.llm.model.generation_config.eos_token_id
|
|
del model.llm.model.config.bos_token_id
|
|
del model.llm.model.config.eos_token_id
|
|
model.llm.model.config.vocab_size = pad_vocab_size
|
|
model.llm.model.config.tie_word_embeddings = False
|
|
model.llm.model.config.use_bias = True
|
|
model.llm.model.save_pretrained(model_path)
|
|
os.system('sed -i s@Qwen2ForCausalLM@CosyVoice2ForCausalLM@g {}/config.json'.format(os.path.abspath(model_path)))
|
|
model.llm.model.config.vocab_size = tmp_vocab_size
|
|
model.llm.model.config.tie_word_embeddings = tmp_tie_embedding
|
|
model.llm.model.set_input_embeddings(embed_tokens)
|