facebook/unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur

unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur

Speech-to-speech translation model from fairseq S2UT (paper/code):

• Spanish-English
• Trained on mTEDx, CoVoST 2, Europarl-ST and VoxPopuli

Usage

import json<br /> import os<br /> from pathlib import Path<br /> import IPython.display as ipd<br /> from fairseq import hub_utils<br /> from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub<br /> from fairseq.models.speech_to_text.hub_interface import S2THubInterface<br /> from fairseq.models.text_to_speech import CodeHiFiGANVocoder<br /> from fairseq.models.text_to_speech.hub_interface import VocoderHubInterface<br /> from huggingface_hub import snapshot_download<br /> import torchaudio<br /> cache_dir = os.getenv("HUGGINGFACE_HUB_CACHE")<br /> #models, cfg, task = load_model_ensemble_and_task_from_hf_hub(<br /> # "facebook/xm_transformer_s2ut_800m-es-en-st-asr-bt_h1_2022",<br /> # arg_overrides={"config_yaml": "config.yaml", "task": "speech_to_text"},<br /> # cache_dir=cache_dir,<br /> # )<br /> # model = models[0].cpu()<br /> # cfg["task"].cpu = True<br /> # generator = task.build_generator([model], cfg)<br /> # # requires 16000Hz mono channel audio<br /> # audio, _ = torchaudio.load("/Users/lpw/git/api-inference-community/docker_images/fairseq/tests/samples/sample2.flac")<br /> # sample = S2THubInterface.get_model_input(task, audio)<br /> # unit = S2THubInterface.get_prediction(task, model, generator, sample)<br /> # speech synthesis<br /> library_name = "fairseq"<br /> cache_dir = (<br /> cache_dir or (Path.home() / ".cache" / library_name).as_posix()<br /> )<br /> cache_dir = snapshot_download(<br /> f"facebook/unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur", cache_dir=cache_dir, library_name=library_name<br /> )<br /> x = hub_utils.from_pretrained(<br /> cache_dir,<br /> "model.pt",<br /> ".",<br /> archive_map=CodeHiFiGANVocoder.hub_models(),<br /> config_yaml="config.json",<br /> fp16=False,<br /> is_vocoder=True,<br /> )<br /> with open(f"{x['args']['data']}/config.json") as f:<br /> vocoder_cfg = json.load(f)<br /> assert (<br /> len(x["args"]["model_path"]) == 1<br /> ), "Too many vocoder models in the input"<br /> vocoder = CodeHiFiGANVocoder(x["args"]["model_path"][0], vocoder_cfg)<br /> tts_model = VocoderHubInterface(vocoder_cfg, vocoder)<br /> tts_sample = tts_model.get_model_input(unit)<br /> wav, sr = tts_model.get_prediction(tts_sample)<br /> ipd.Audio(wav, rate=sr)<br />