Text to speech
Text to speech turns written text into spoken audio. It powers voice replies, read-aloud, narration and dubbing.
With no arguments, createTextToSpeech runs
Reverb, a voice
cloning model: it speaks the text in the voice of a short reference clip,
in English, German, French, Spanish, Italian, Polish or Portuguese.
Quick start
Section titled “Quick start”Reverb expects text written the way it is spoken, so the example passes
verbalize with Verbalis
(npm i verbalis) to read numbers, dates and symbols as words.
import { function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';import { class Verbalizer
Verbalizer } from 'verbalis';
// 1. Load the model once. The weights download from the Hugging Face Hub.const const tts: TextToSpeech
tts = await function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech({ ...const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.textToSpeech: { readonly REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; };}
textToSpeech.type REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; };}
REVERB.type DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string;}
DEFAULT, TextToSpeechModel.verbalize?: ((text: string, language: ReverbLanguage) => string) | undefined
verbalize: (text: string
text, language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language) => new new Verbalizer(language: string, options?: Options): Verbalizer
Verbalizer(language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language).Verbalizer.process(text: string): string
process(text: string
text),});
// 2. Speak a text in a voice: mono samples at tts.sampleRate come back.const const audio: Float32Array<ArrayBufferLike>
audio = await const tts: TextToSpeech
tts.TextToSpeech.synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>
synthesize('It is 21 °C today.', const voice: Voice
voice);
// 3. Free the GPU memory when done.const tts: TextToSpeech
tts.TextToSpeech.dispose(): void
dispose();The engine must be set up first, see Getting started.
Methods
Section titled “Methods”function createTextToSpeech( model?: TextToSpeechModel, options?: LoadOptions,): Promise<TextToSpeech>;
interface TextToSpeech { readonly sampleRate: number; synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>; dispose(): void;}
interface SynthesizeOptions { readonly steps?: number; // default 8 readonly seed?: number; // default 0 readonly raw?: boolean; // default false}synthesize(text, voice, options?)speakstextinvoiceand returns mono samples atsampleRate. The length of the clip follows from the text and the pace of the reference. Calls run one after another. A text too long for one pass on the device throwsINVALID_ARGUMENT, so split long texts.- steps - how many passes pick the codes.
- seed - an integer that seeds the random choices. The same text, voice and seed give the same audio.
- raw - keeps the clip as decoded. By default the silence at both ends is trimmed, to 100 ms.
sampleRateis the rate of the returned samples, 24000 for Reverb. Pass it to anAudioBufferto play them.dispose()frees the GPU memory.
Voices
Section titled “Voices”A voice is a reference clip of the speaker, at least 3 seconds long, as Reverb reads it: the clip’s Higgs Audio v2 codec codes, its transcript and its language.
interface Voice { readonly codes: Int32Array; readonly frames: number; readonly transcript: string; readonly language: ReverbLanguage; readonly speechFrames?: number;}
type ReverbLanguage = 'en' | 'de' | 'fr' | 'es' | 'it' | 'pl' | 'pt';- codes - the codec codes, 8 codebooks ×
frames, codebook-major: codetof codebookcatc * frames + t. Each is in [0, 1024). - frames - the clip’s length in codec frames, 25 per second. Under 75
(3 s) throws
INVALID_ARGUMENT. - transcript - what is said in the clip, word for word.
- language - the clip’s language, an ISO 639-1 code. Any other value
throws
INVALID_ARGUMENT. - speechFrames - optional. How many of the clip’s frames the speaker
spends speaking, with the silent edges dropped and pauses capped at 0.5 s.
The output’s pace follows it; without it, the whole clip counts.
fromRecordingsets it. Not an integer of at least 1 throwsINVALID_ARGUMENT.
createVoiceMaker makes a voice from a recording. It takes the same model
config as createTextToSpeech and reads only its codecPath: the codec’s
encoder, about 170 MB, run in f32.
import { function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech, function createVoiceMaker(config?: TextToSpeechModel, opts?: LoadOptions): Promise<VoiceMaker>
createVoiceMaker, function decodeAudio(data: ArrayBuffer, sampleRate?: number): Promise<Float32Array>
decodeAudio, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';import { class Verbalizer
Verbalizer } from 'verbalis';
const const model: { verbalize: (text: string, language: string) => string; modelPath: string; codecPath: string; tokenizerPath: string;}
model = { ...const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.textToSpeech: { readonly REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; };}
textToSpeech.type REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; };}
REVERB.type DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string;}
DEFAULT, verbalize: (text: string, language: string) => string
verbalize: (text: string
text: string, language: string
language: string) => new new Verbalizer(language: string, options?: Options): Verbalizer
Verbalizer(language: string
language).Verbalizer.process(text: string): string
process(text: string
text),};
// 1. Make the voice from a recording decoded at 24 kHz.const const maker: VoiceMaker
maker = await function createVoiceMaker(config?: TextToSpeechModel, opts?: LoadOptions): Promise<VoiceMaker>
createVoiceMaker(const model: { verbalize: (text: string, language: string) => string; modelPath: string; codecPath: string; tokenizerPath: string;}
model);const const voice: Voice
voice = await const maker: VoiceMaker
maker.VoiceMaker.fromRecording(samples: Float32Array, options: FromRecordingOptions): Promise<Voice>
fromRecording(await function decodeAudio(data: ArrayBuffer, sampleRate?: number): Promise<Float32Array>
decodeAudio(await const file: File
file.Blob.arrayBuffer(): Promise<ArrayBuffer>
arrayBuffer(), 24_000), { FromRecordingOptions.sampleRate: number
sampleRate: 24_000, FromRecordingOptions.transcript: string
transcript: 'Hi, this is my voice for the demo.', FromRecordingOptions.language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language: 'en',});const maker: VoiceMaker
maker.VoiceMaker.dispose(): void
dispose();
// 2. Speak in it, as often as you like.const const tts: TextToSpeech
tts = await function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech(const model: { verbalize: (text: string, language: string) => string; modelPath: string; codecPath: string; tokenizerPath: string;}
model);const const audio: Float32Array<ArrayBufferLike>
audio = await const tts: TextToSpeech
tts.TextToSpeech.synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>
synthesize('The weather today is lovely.', const voice: Voice
voice);function createVoiceMaker(model?: TextToSpeechModel, options?: LoadOptions): Promise<VoiceMaker>;
interface VoiceMaker { readonly minSeconds: number; readonly maxSeconds: number; fromRecording(samples: Float32Array, options: FromRecordingOptions): Promise<Voice>; dispose(): void;}
interface FromRecordingOptions { readonly sampleRate: number; readonly transcript: string; readonly language: ReverbLanguage;}fromRecording(samples, options)returns the voice of monosamplesatsampleRate, with theirtranscriptandlanguageas above. Calls run one after another.minSecondsandmaxSecondsare the shortest and the longest recording one call takes: 3 s, and about 20 s depending on the device. Any other throwsINVALID_ARGUMENT.dispose()frees the GPU memory.synthesizedoes not need the maker.
Options
Section titled “Options”The first argument is the model config, a plain object with the fields
below. models.textToSpeech.REVERB holds ready-made ones: INT4 (int4
weights, the DEFAULT), INT8 (int8 weights), F16 (f16 weights, f16
forced) and F32 (f32 weights, f32 forced). Pass one as is, copy it with a
field changed, or point at your own files.
interface TextToSpeechModel { readonly modelPath?: ModelPath; readonly codecPath?: ModelPath; readonly tokenizerPath?: string; readonly dtype?: 'f32' | 'f16'; readonly verbalize?: (text: string, language: ReverbLanguage) => string;}
type ModelPath = string | RangeSource; // a URL, or bytes you already haveExample:
import { function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech } from 'runntime/zoo';
// Your own copy of the files.await function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech({ TextToSpeechModel.modelPath?: ModelPath | undefined
modelPath: '/models/reverb/model.safetensors', TextToSpeechModel.codecPath?: ModelPath | undefined
codecPath: '/models/higgs-audio-v2-tokenizer/model.safetensors', TextToSpeechModel.tokenizerPath?: string | undefined
tokenizerPath: '/models/reverb/tokenizer.json',});- modelPath - the Reverb weights, a
model.safetensorsfile: a hosted int4, int8, f16 or f32 file, or your own copy of one. The weight format is read from the file. - codecPath - the codec weights, a
model.safetensorsfile: the hosted one or the Higgs Audio v2 tokenizer’s from the Hub.createTextToSpeechreads its decoder,createVoiceMakerits encoder. - tokenizerPath - Reverb’s
tokenizer.json. - dtype -
'f16'or'f32', the precision of the math and the float weights.f16needs a device withshader-f16. Default:f16where the device has it,f32elsewhere. Both work with any weight file. - verbalize - rewrites the text and the voice’s transcript the way
they are read aloud before
synthesizespeaks, in the voice’s language: numbers, dates and symbols as words, as Verbalis does in the examples above. Without it both go to the model as written, digits and symbols included. An empty result throwsINVALID_ARGUMENT, and an error it throws rejectssynthesizeas is.
Every field is optional. A missing path comes from REVERB.DEFAULT.
Loading options
Section titled “Loading options”The second argument controls the download. Every field is optional.
interface LoadOptions { cache?: WeightCache; cacheId?: string; onProgress?: (name: string, doneBytes: number, totalBytes: number) => void; onBytes?: (chunkBytes: number) => void; signal?: AbortSignal;}Example:
import { function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache, function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';
const const tts: TextToSpeech
tts = await function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech(const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.textToSpeech: { readonly REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; };}
textToSpeech.type REVERB: { readonly DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT4: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly INT8: { modelPath: string; codecPath: string; tokenizerPath: string; }; readonly F16: { readonly dtype: "f16"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; }; readonly F32: { readonly dtype: "f32"; readonly modelPath: string; readonly codecPath: string; readonly tokenizerPath: string; };}
REVERB.type DEFAULT: { modelPath: string; codecPath: string; tokenizerPath: string;}
DEFAULT, { LoadOptions.cache?: WeightCache | undefined
cache: await function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache('my-app'), LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress: (name: string
name, done: number
done, total: number
total) => var console: Console
console.Console.log(...data: any[]): void
log(`${var Math: Math
Math.Math.round(x: number): number
round((100 * done: number
done) / total: number
total)}%`),});- cache - keeps the downloaded weights and tokenizer in the browser, so
the next visit loads them without the network.
createOpfsCache(name)is the built-in one. - onProgress - called while the weights upload to the GPU, with the tensor name, bytes done and bytes total, the model and codec files counted together. Use it for a progress bar.
- signal - an
AbortSignal. Abort it to stop the load between steps. A step already running, like the download or the GPU upload, finishes first.
Supported models
Section titled “Supported models”| Model | Config | Languages | Parameters | Weights |
|---|---|---|---|---|
| Reverb | models.textToSpeech.REVERB | en, de, fr, es, it, pl, pt | 297M | int4 191 MB, int8 307 MB, f16 594 MB, f32 1.19 GB, plus 26 MB of the codec file |