Skip to content

Text to speech

Text to speech turns written text into spoken audio. It powers voice replies, read-aloud, narration and dubbing.

With no arguments, createTextToSpeech runs Reverb, a voice cloning model: it speaks the text in the voice of a short reference clip, in English, German, French, Spanish, Italian, Polish or Portuguese.

Reverb expects text written the way it is spoken, so the example passes verbalize with Verbalis (npm i verbalis) to read numbers, dates and symbols as words.

import {
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
import {
class Verbalizer
Verbalizer
} from 'verbalis';
// 1. Load the model once. The weights download from the Hugging Face Hub.
const
const tts: TextToSpeech
tts
= await
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
({
...
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
textToSpeech: {
readonly REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
};
}
textToSpeech
.
type REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
}
REVERB
.
type DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
DEFAULT
,
TextToSpeechModel.verbalize?: ((text: string, language: ReverbLanguage) => string) | undefined
verbalize
: (
text: string
text
,
language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language
) => new
new Verbalizer(language: string, options?: Options): Verbalizer
Verbalizer
(
language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language
).
Verbalizer.process(text: string): string
process
(
text: string
text
),
});
// 2. Speak a text in a voice: mono samples at tts.sampleRate come back.
const
const audio: Float32Array<ArrayBufferLike>
audio
= await
const tts: TextToSpeech
tts
.
TextToSpeech.synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>
synthesize
('It is 21 °C today.',
const voice: Voice
voice
);
// 3. Free the GPU memory when done.
const tts: TextToSpeech
tts
.
TextToSpeech.dispose(): void
dispose
();

The engine must be set up first, see Getting started.

function createTextToSpeech(
model?: TextToSpeechModel,
options?: LoadOptions,
): Promise<TextToSpeech>;
interface TextToSpeech {
readonly sampleRate: number;
synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>;
dispose(): void;
}
interface SynthesizeOptions {
readonly steps?: number; // default 8
readonly seed?: number; // default 0
readonly raw?: boolean; // default false
}
  • synthesize(text, voice, options?) speaks text in voice and returns mono samples at sampleRate. The length of the clip follows from the text and the pace of the reference. Calls run one after another. A text too long for one pass on the device throws INVALID_ARGUMENT, so split long texts.
    • steps - how many passes pick the codes.
    • seed - an integer that seeds the random choices. The same text, voice and seed give the same audio.
    • raw - keeps the clip as decoded. By default the silence at both ends is trimmed, to 100 ms.
  • sampleRate is the rate of the returned samples, 24000 for Reverb. Pass it to an AudioBuffer to play them.
  • dispose() frees the GPU memory.

A voice is a reference clip of the speaker, at least 3 seconds long, as Reverb reads it: the clip’s Higgs Audio v2 codec codes, its transcript and its language.

interface Voice {
readonly codes: Int32Array;
readonly frames: number;
readonly transcript: string;
readonly language: ReverbLanguage;
readonly speechFrames?: number;
}
type ReverbLanguage = 'en' | 'de' | 'fr' | 'es' | 'it' | 'pl' | 'pt';
  • codes - the codec codes, 8 codebooks × frames, codebook-major: code t of codebook c at c * frames + t. Each is in [0, 1024).
  • frames - the clip’s length in codec frames, 25 per second. Under 75 (3 s) throws INVALID_ARGUMENT.
  • transcript - what is said in the clip, word for word.
  • language - the clip’s language, an ISO 639-1 code. Any other value throws INVALID_ARGUMENT.
  • speechFrames - optional. How many of the clip’s frames the speaker spends speaking, with the silent edges dropped and pauses capped at 0.5 s. The output’s pace follows it; without it, the whole clip counts. fromRecording sets it. Not an integer of at least 1 throws INVALID_ARGUMENT.

createVoiceMaker makes a voice from a recording. It takes the same model config as createTextToSpeech and reads only its codecPath: the codec’s encoder, about 170 MB, run in f32.

import {
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
,
function createVoiceMaker(config?: TextToSpeechModel, opts?: LoadOptions): Promise<VoiceMaker>
createVoiceMaker
,
function decodeAudio(data: ArrayBuffer, sampleRate?: number): Promise<Float32Array>
decodeAudio
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
import {
class Verbalizer
Verbalizer
} from 'verbalis';
const
const model: {
verbalize: (text: string, language: string) => string;
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
model
= {
...
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
textToSpeech: {
readonly REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
};
}
textToSpeech
.
type REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
}
REVERB
.
type DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
DEFAULT
,
verbalize: (text: string, language: string) => string
verbalize
: (
text: string
text
: string,
language: string
language
: string) => new
new Verbalizer(language: string, options?: Options): Verbalizer
Verbalizer
(
language: string
language
).
Verbalizer.process(text: string): string
process
(
text: string
text
),
};
// 1. Make the voice from a recording decoded at 24 kHz.
const
const maker: VoiceMaker
maker
= await
function createVoiceMaker(config?: TextToSpeechModel, opts?: LoadOptions): Promise<VoiceMaker>
createVoiceMaker
(
const model: {
verbalize: (text: string, language: string) => string;
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
model
);
const
const voice: Voice
voice
= await
const maker: VoiceMaker
maker
.
VoiceMaker.fromRecording(samples: Float32Array, options: FromRecordingOptions): Promise<Voice>
fromRecording
(await
function decodeAudio(data: ArrayBuffer, sampleRate?: number): Promise<Float32Array>
decodeAudio
(await
const file: File
file
.
Blob.arrayBuffer(): Promise<ArrayBuffer>
arrayBuffer
(), 24_000), {
FromRecordingOptions.sampleRate: number
sampleRate
: 24_000,
FromRecordingOptions.transcript: string
transcript
: 'Hi, this is my voice for the demo.',
FromRecordingOptions.language: "en" | "de" | "fr" | "es" | "it" | "pl" | "pt"
language
: 'en',
});
const maker: VoiceMaker
maker
.
VoiceMaker.dispose(): void
dispose
();
// 2. Speak in it, as often as you like.
const
const tts: TextToSpeech
tts
= await
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
(
const model: {
verbalize: (text: string, language: string) => string;
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
model
);
const
const audio: Float32Array<ArrayBufferLike>
audio
= await
const tts: TextToSpeech
tts
.
TextToSpeech.synthesize(text: string, voice: Voice, options?: SynthesizeOptions): Promise<Float32Array>
synthesize
('The weather today is lovely.',
const voice: Voice
voice
);
function createVoiceMaker(model?: TextToSpeechModel, options?: LoadOptions): Promise<VoiceMaker>;
interface VoiceMaker {
readonly minSeconds: number;
readonly maxSeconds: number;
fromRecording(samples: Float32Array, options: FromRecordingOptions): Promise<Voice>;
dispose(): void;
}
interface FromRecordingOptions {
readonly sampleRate: number;
readonly transcript: string;
readonly language: ReverbLanguage;
}
  • fromRecording(samples, options) returns the voice of mono samples at sampleRate, with their transcript and language as above. Calls run one after another.
  • minSeconds and maxSeconds are the shortest and the longest recording one call takes: 3 s, and about 20 s depending on the device. Any other throws INVALID_ARGUMENT.
  • dispose() frees the GPU memory. synthesize does not need the maker.

The first argument is the model config, a plain object with the fields below. models.textToSpeech.REVERB holds ready-made ones: INT4 (int4 weights, the DEFAULT), INT8 (int8 weights), F16 (f16 weights, f16 forced) and F32 (f32 weights, f32 forced). Pass one as is, copy it with a field changed, or point at your own files.

interface TextToSpeechModel {
readonly modelPath?: ModelPath;
readonly codecPath?: ModelPath;
readonly tokenizerPath?: string;
readonly dtype?: 'f32' | 'f16';
readonly verbalize?: (text: string, language: ReverbLanguage) => string;
}
type ModelPath = string | RangeSource; // a URL, or bytes you already have

Example:

import {
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
} from 'runntime/zoo';
// Your own copy of the files.
await
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
({
TextToSpeechModel.modelPath?: ModelPath | undefined
modelPath
: '/models/reverb/model.safetensors',
TextToSpeechModel.codecPath?: ModelPath | undefined
codecPath
: '/models/higgs-audio-v2-tokenizer/model.safetensors',
TextToSpeechModel.tokenizerPath?: string | undefined
tokenizerPath
: '/models/reverb/tokenizer.json',
});
  • modelPath - the Reverb weights, a model.safetensors file: a hosted int4, int8, f16 or f32 file, or your own copy of one. The weight format is read from the file.
  • codecPath - the codec weights, a model.safetensors file: the hosted one or the Higgs Audio v2 tokenizer’s from the Hub. createTextToSpeech reads its decoder, createVoiceMaker its encoder.
  • tokenizerPath - Reverb’s tokenizer.json.
  • dtype - 'f16' or 'f32', the precision of the math and the float weights. f16 needs a device with shader-f16. Default: f16 where the device has it, f32 elsewhere. Both work with any weight file.
  • verbalize - rewrites the text and the voice’s transcript the way they are read aloud before synthesize speaks, in the voice’s language: numbers, dates and symbols as words, as Verbalis does in the examples above. Without it both go to the model as written, digits and symbols included. An empty result throws INVALID_ARGUMENT, and an error it throws rejects synthesize as is.

Every field is optional. A missing path comes from REVERB.DEFAULT.

The second argument controls the download. Every field is optional.

interface LoadOptions {
cache?: WeightCache;
cacheId?: string;
onProgress?: (name: string, doneBytes: number, totalBytes: number) => void;
onBytes?: (chunkBytes: number) => void;
signal?: AbortSignal;
}

Example:

import {
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache
,
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
const
const tts: TextToSpeech
tts
= await
function createTextToSpeech(config?: TextToSpeechModel, opts?: LoadOptions): Promise<TextToSpeech>
createTextToSpeech
(
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
textToSpeech: {
readonly REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
};
}
textToSpeech
.
type REVERB: {
readonly DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT4: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly INT8: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
};
readonly F16: {
readonly dtype: "f16";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
readonly F32: {
readonly dtype: "f32";
readonly modelPath: string;
readonly codecPath: string;
readonly tokenizerPath: string;
};
}
REVERB
.
type DEFAULT: {
modelPath: string;
codecPath: string;
tokenizerPath: string;
}
DEFAULT
, {
LoadOptions.cache?: WeightCache | undefined
cache
: await
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache
('my-app'),
LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress
: (
name: string
name
,
done: number
done
,
total: number
total
) =>
var console: Console
console
.
Console.log(...data: any[]): void
log
(`${
var Math: Math
Math
.
Math.round(x: number): number
round
((100 *
done: number
done
) /
total: number
total
)}%`),
});
  • cache - keeps the downloaded weights and tokenizer in the browser, so the next visit loads them without the network. createOpfsCache(name) is the built-in one.
  • onProgress - called while the weights upload to the GPU, with the tensor name, bytes done and bytes total, the model and codec files counted together. Use it for a progress bar.
  • signal - an AbortSignal. Abort it to stop the load between steps. A step already running, like the download or the GPU upload, finishes first.
ModelConfigLanguagesParametersWeights
Reverbmodels.textToSpeech.REVERBen, de, fr, es, it, pl, pt297Mint4 191 MB, int8 307 MB, f16 594 MB, f32 1.19 GB, plus 26 MB of the codec file