Vyvo Multilingual v0.1

--- license: apache-2.0 language: - en - ja pipeline_tag: text-to-speech tags: - text-to-speech - voice-cloning - multilingual --- # Vyvo-Multilingual-v0.1 Multilingual (English + Japanese) voice-cloning TTS. Give a short reference clip and a target text; the model speaks the text in the reference voice. - **Backbone:** Qwen3-0.6B - **Audio codec:** [kyutai/mimi](https://huggingface.co/kyutai/mimi) (32 codebooks, 24 kHz) - **Use a reference clip** for best results. ## Usage ```python import soundfile as sf import torch import torchaudio from transformers import (AutoModelForCausalLM, AutoTokenizer, AutoFeatureExtractor, MimiModel) REPO = "Vyvo/Vyvo-Multilingual-v0.1" DEVICE = "cuda" # Token layout (must match training) BASE = 151669 # Qwen3 base vocab; audio ids start above it NUM_CODEBOOKS = 32 CODEBOOK_SIZE = 2048 AUDIO_OFFSET = 10 SOS, EOS, SOH, EOH, SOA = 1, 2, 3, 4, 5 # special tokens, as offsets above BASE # Load the model, tokenizer, and the Mimi codec tokenizer = AutoTokenizer.from_pretrained(REPO) model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.bfloat16).to(DEVICE).eval() mimi = MimiModel.from_pretrained("kyutai/mimi").to(DEVICE).eval() feature_extractor = AutoFeatureExtractor.from_pretrained("kyutai/mimi") def encode_reference(path): # Load a wav and encode it to Mimi audio tokens (as LM token ids). wav, sr = sf.read(path, dtype="float32", always_2d=False) if wav.ndim > 1: wav = wav.mean(axis=1) target_sr = feature_extractor.sampling_rate if sr != target_sr: wav = torchaudio.functional.resample( torch.from_numpy(wav).unsqueeze(0), sr, target_sr).squeeze(0).numpy() inputs = feature_extractor(raw_audio=wav, sampling_rate=target_sr, return_tensors="pt") codes = mimi.encode(inputs["input_values"].to(DEVICE), num_quantizers=NUM_CODEBOOKS).audio_codes[0].cpu() # (codebooks, frames) frame_interleaved = codes.transpose(0, 1).reshape(-1).tolist() return [code + AUDIO_OFFSET + (i % NUM_CODEBOOKS) * CODEBOOK_SIZE + BASE for i, code in enumerate(frame_interleaved)] def build_prompt(reference_tokens, reference_text, target_text): # [SOH] ref_text + target_text [eot] [EOH] [SOA] [SOS] <reference audio> text_ids = tokenizer(reference_text + " " + target_text, add_special_tokens=False).input_ids head = [BASE + SOH] + text_ids + [tokenizer.eos_token_id, BASE + EOH, BASE + SOA, BASE + SOS] return head + reference_to...