Skip to content

Image classification

Image classification assigns an image to one of a fixed set of classes. You pass in the pixels and get back the classes ranked by probability: what the image most likely shows, and how sure the model is of each answer. Unlike object detection, which locates several objects with bounding boxes, classification labels the image as a whole.

createImageClassifier runs MobileNetV4 Conv-Small, trained on ImageNet-1k: 1000 classes of animals, vehicles, food, tools and household objects.

import {
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
,
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
// 1. Load the model once. The weights download from the Hugging Face Hub.
const
const classifier: ImageClassifier
classifier
= await
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
(
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
}
imageClassification
.
type MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
}
MOBILENETV4
.
type DEFAULT: {
modelPath: string;
}
DEFAULT
);
// 2. Get the pixels of an image, here from a canvas the picture is drawn on.
const
const ctx: CanvasRenderingContext2D
ctx
=
const canvas: HTMLCanvasElement
canvas
.
HTMLCanvasElement.getContext(contextId: "2d", options?: CanvasRenderingContext2DSettings): CanvasRenderingContext2D | null (+5 overloads)
getContext
('2d')!;
const
const image: ImageBuffer
image
=
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData
(
const ctx: CanvasRenderingContext2D
ctx
.
CanvasImageData.getImageData(sx: number, sy: number, sw: number, sh: number, settings?: ImageDataSettings): ImageData
getImageData
(0, 0,
const canvas: HTMLCanvasElement
canvas
.
HTMLCanvasElement.width: number
width
,
const canvas: HTMLCanvasElement
canvas
.
HTMLCanvasElement.height: number
height
));
// 3. Classify it.
const
const classes: (Classification & {
label: string;
})[]
classes
= await
const classifier: ImageClassifier
classifier
.
ImageClassifier.classify(image: ImageBuffer, options?: ClassifyOptions): Promise<(Classification & {
label: string;
})[]>
classify
(
const image: ImageBuffer
image
, {
ClassifyOptions.topk?: number | undefined
topk
: 5 });
// 4. Use the result: the five most likely classes, best first.
for (const {
const label: string
label
,
const confidence: number
confidence
} of
const classes: (Classification & {
label: string;
})[]
classes
) {
var console: Console
console
.
Console.log(...data: any[]): void
log
(
const label: string
label
,
const confidence: number
confidence
);
}
// minibus 0.59
// police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria 0.11
// trolleybus, trolley coach, trackless trolley 0.07
// ...
// 5. Free the GPU memory when done.
const classifier: ImageClassifier
classifier
.
ImageClassifier.dispose(): void
dispose
();

The example reads one picture off a canvas. A video or a camera works the same way: draw each frame on the canvas and call classify again, the classifier is loaded once.

The engine must be set up first, see Getting started. The model runs in half precision, so the device needs shader-f16.

Your camera, frame by frame through classify() on your GPU, with the five most likely classes under each frame. Point it at a cup, a keyboard, a dog. The dashed square is what the model sees: everything outside it is cropped away. No camera? The sample picture goes through the same call.

classify returns the classes best first: the topk most likely ones, or all 1000 when topk is not given. Each entry is:

interface Classification {
label: string; // the class name: 'minibus', 'golden retriever', ...
classId: number; // the same as a number, index into classifier.labels
confidence: number; // how sure the model is, 0 to 1
}

The confidences of all 1000 classes sum to 1. A low top score means the model is torn between a few similar classes: in the example above the next guesses are all vans and buses too.

function createImageClassifier(
model?: ImageClassifierModel,
options?: LoadOptions,
): Promise<ImageClassifier>;
interface ImageClassifier {
readonly labels: readonly string[];
readonly inputSize: number;
readonly schemas: {
readonly input: ClassifyInput;
readonly output: (topk: number) => ClassificationList;
};
classify(
image: ImageBuffer,
options?: ClassifyOptions,
): Promise<(Classification & { label: string })[]>;
encodeClassify(options: EncodeClassifyOptions): void;
dispose(): void;
}
interface ClassifyOptions {
readonly topk?: number; // default: every class
}
  • classify(image, options?) returns the most likely classes, best first. image is an ImageBuffer of any size. Build one with imageBufferFromImageData(imageData) from a canvas, or imageBuffer(bytes, width, height, format?) from raw rgba, rgb, bgra, bgr or gray bytes. topk caps how many come back.
  • encodeClassify(options) reads an image already on the GPU and writes the best classes to a GPU buffer, see GPU in, GPU out.
  • labels are the class names, by class index.
  • inputSize is the model input width and height, 224.
  • schemas are the TypeGPU layouts of the model input and of the list encodeClassify writes, for buffers passed to it.
  • dispose() frees the GPU memory.

When the image is already on the GPU, a video frame or the output of your own shader, encodeClassify skips the CPU: no pixels are read back, and the best classes are ranked on the GPU, straight into a buffer you own, for your next pass to read.

import {
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
,
function initRunntime(root: TgpuRoot): RunntimeExecutor
initRunntime
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
import
const tgpu: {
const: {
<TDataType extends AnyData>(dataType: TDataType, value: InferGPU<TDataType>): TgpuConst<TDataType>;
<TElement extends AnyWgslData>(dataType: (elementCount: number) => WgslArray<TElement>, value: InferGPU<WgslArray<TElement>>): TgpuConst<WgslArray<TElement>>;
};
fn: {
<Args extends BaseData[] | []>(argTypes: Args, returnType?: undefined): TgpuFnShell<Args, Void>;
<Args extends BaseData[] | [], Return extends BaseData>(argTypes: Args, returnType: Return): TgpuFnShell<Args, Return>;
<T extends AnyFn>(inner: T): TgpuGenericFn<T>;
};
... 17 more ...;
'~unstable': typeof import("/home/runner/work/runntime/runntime/node_modules/.pnpm/typegpu@0.12.5/node_modules/typegpu/tgpuUnstable");
}
tgpu
from 'typegpu';
// 1. Set up the engine and load the model once.
const
const root: TgpuRoot
root
= await
const tgpu: {
const: {
<TDataType extends AnyData>(dataType: TDataType, value: InferGPU<TDataType>): TgpuConst<TDataType>;
<TElement extends AnyWgslData>(dataType: (elementCount: number) => WgslArray<TElement>, value: InferGPU<WgslArray<TElement>>): TgpuConst<WgslArray<TElement>>;
};
fn: {
<Args extends BaseData[] | []>(argTypes: Args, returnType?: undefined): TgpuFnShell<Args, Void>;
<Args extends BaseData[] | [], Return extends BaseData>(argTypes: Args, returnType: Return): TgpuFnShell<Args, Return>;
<T extends AnyFn>(inner: T): TgpuGenericFn<T>;
};
... 17 more ...;
'~unstable': typeof import("/home/runner/work/runntime/runntime/node_modules/.pnpm/typegpu@0.12.5/node_modules/typegpu/tgpuUnstable");
}
tgpu
.
init: (options?: InitOptions) => Promise<TgpuRoot>
init
({
device?: (GPUDeviceDescriptor & {
optionalFeatures?: Iterable<GPUFeatureName>;
}) | undefined
device
: {
optionalFeatures?: Iterable<GPUFeatureName> | undefined
optionalFeatures
: ['subgroups', 'shader-f16'] } });
function initRunntime(root: TgpuRoot): RunntimeExecutor
initRunntime
(
const root: TgpuRoot
root
);
const
const classifier: ImageClassifier
classifier
= await
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
(
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
}
imageClassification
.
type MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
}
MOBILENETV4
.
type DEFAULT: {
modelPath: string;
}
DEFAULT
);
// 2. A buffer for up to 5 classes, typed by the classifier's schema.
const
const best: TgpuBuffer<ClassificationList> & StorageFlag
best
=
const root: TgpuRoot
root
.
TgpuRoot.createBuffer<ClassificationList>(typeSchema: ClassificationList, initial?: ((buffer: TgpuBuffer<NoInfer<ClassificationList>>) => void) | InferInput<NoInfer<ClassificationList>> | undefined): TgpuBuffer<ClassificationList> (+1 overload)
createBuffer
(
const classifier: ImageClassifier
classifier
.
ImageClassifier.schemas: {
readonly input: ClassifyInput;
readonly output: (topk: number) => ClassificationList;
}
schemas
.
output: (topk: number) => ClassificationList
output
(5)).
TgpuBuffer<ClassificationList>.$usage<["storage"]>(usages_0: "storage"): TgpuBuffer<ClassificationList> & StorageFlag
$usage
('storage');
// 3. Each frame: video in, the list out, then your own passes in the same encoder.
const
const encoder: TgpuCommandEncoder
encoder
=
const root: TgpuRoot
root
['~unstable'].
function createCommandEncoder(descriptor?: GPUCommandEncoderDescriptor): TgpuCommandEncoder
createCommandEncoder
();
const classifier: ImageClassifier
classifier
.
ImageClassifier.encodeClassify(options: EncodeClassifyOptions): void
encodeClassify
({
EncodeOptions<GpuImage | GpuTensor<ClassifyInput>, GpuTensor<ClassificationList>>.source: GpuImage | GpuTensor<ClassifyInput>
source
:
const root: TgpuRoot
root
.
TgpuRoot.device: GPUDevice
device
.
GPUDevice.importExternalTexture(descriptor: GPUExternalTextureDescriptor): GPUExternalTexture
importExternalTexture
({
GPUExternalTextureDescriptor.source: HTMLVideoElement | VideoFrame
source
:
const video: HTMLVideoElement
video
}),
EncodeOptions<GpuImage | GpuTensor<ClassifyInput>, GpuTensor<ClassificationList>>.target: GpuTensor<ClassificationList>
target
:
const best: TgpuBuffer<ClassificationList> & StorageFlag
best
,
EncodeOptions<GpuImage | GpuTensor<ClassifyInput>, GpuTensor<ClassificationList>>.encoder?: GpuEncoder | undefined
encoder
,
});
// ...record your own passes that read `best` here...
const encoder: TgpuCommandEncoder
encoder
.
TgpuCommandEncoder.submit(): void
submit
();

Source - one of:

  • a texture: a GPUTexture, a TypeGPU texture with sampled usage, or a GPUExternalTexture from a video. It is fitted like an ImageBuffer, with the classifier’s resizeMode, and each value 0-1 counts as a byte 0-255. Formats: rgba8unorm, bgra8unorm, rgb10a2unorm, rgba16float, rgba32float. sRGB formats are rejected: the GPU would turn their values linear, which the model was not trained on.
  • a buffer holding the model input as is: 3×224×224 f32, normalized RGB, one plane per channel. A raw GPUBuffer needs COPY_SRC usage.

Target - a buffer with STORAGE usage, laid out as schemas.output(topk):

struct Classification { classId: u32, confidence: f32 }
array<Classification, topk>

The entries are the classes classify would return, best first: classId indexes labels, confidence is 0 to 1, a softmax over every class. In a raw GPUBuffer entry i starts at byte 8 * i.

topk caps how many are written, by default as many as the target holds, up to every class.

encoder - optional. Pass a GPUCommandEncoder or a TypeGPU command encoder to record the classification together with your own passes into one command buffer. You then finish and submit it yourself, before dispose(). Without it, the call submits its own before it returns, and does not wait for the GPU. Work you submit after it sees the result.

Every resource must come from the device passed to initRunntime(). A wrong format, usage or size, or an option out of range, throws INVALID_ARGUMENT.

The first argument is the model config, a plain object with the fields below. Every field is optional. models.imageClassification has the ready-made config, so the quick start passes it as is.

interface ImageClassifierModel {
readonly modelPath?: ModelPath;
readonly labels?: readonly string[]; // 1000 names, ImageNet by default
readonly resizeMode?: 'crop' | 'stretch' | 'letterbox'; // default 'crop'
}
type ModelPath = string | RangeSource; // a URL, or bytes you already have

Examples:

// Weights hosted by you, class names in another language, no cropping.
await
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
({
ImageClassifierModel.modelPath?: ModelPath | undefined
modelPath
: '/models/mobilenetv4/model.safetensors',
ImageClassifierModel.labels?: readonly string[] | undefined
labels
:
const myLabels: string[]
myLabels
,
ImageClassifierModel.resizeMode?: ResizeMode | undefined
resizeMode
: 'stretch',
});
  • modelPath - the weights, a model.safetensors file of MobileNetV4 with the 1000 ImageNet classes. Default: the file on the Hugging Face Hub, so there is nothing to host. Point it at your own copy to serve the weights yourself.
  • labels - the class names, one per class, 1000 of them. Default: the ImageNet names in English.
  • resizeMode - how the image fits the model input. crop (default) cuts the middle 87.5% of the centered square, the crop the model was evaluated with; stretch distorts, letterbox pads.

The second argument controls the download and the warm-up run. Every field is optional.

interface LoadOptions {
cache?: WeightCache;
cacheId?: string;
onProgress?: (name: string, doneBytes: number, totalBytes: number) => void;
onBytes?: (chunkBytes: number) => void;
signal?: AbortSignal;
warmUp?: boolean;
}

By default create runs the model once on a blank input before it returns, so the kernels compile during the load and a GPU error fails the load. With warmUp: false it skips that run and returns sooner. The first call then pays the compile time, and a GPU error is not reported: the call returns zeros instead of throwing.

Example:

import {
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
,
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache
,
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
} from 'runntime/zoo';
const
const classifier: ImageClassifier
classifier
= await
function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>
createImageClassifier
(
const models: {
readonly embedding: {
readonly EMBEDDINGGEMMA_2: {
DEFAULT: {
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F16: {
dtype: "f16";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
F32: {
dtype: "f32";
modelPath: string;
visionPath: string;
audioPath: string;
tokenizerPath: string;
};
};
readonly ALL_MINILM_L6_V2: {
DEFAULT: {
tokenizerPath: string;
} & {
modelPath: string;
};
F16: {
tokenizerPath: string;
} & {
dtype: "f16";
modelPath: string;
};
F32: {
tokenizerPath: string;
} & {
modelPath: string;
dtype: "f32";
};
};
};
... 7 more ...;
readonly imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
};
}
models
.
imageClassification: {
readonly MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
};
}
imageClassification
.
type MOBILENETV4: {
readonly DEFAULT: {
modelPath: string;
};
readonly SMALL: {
modelPath: string;
};
}
MOBILENETV4
.
type DEFAULT: {
modelPath: string;
}
DEFAULT
, {
LoadOptions.cache?: WeightCache | undefined
cache
: await
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache
('my-app'),
LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress
: (
name: string
name
,
done: number
done
,
total: number
total
) =>
var console: Console
console
.
Console.log(...data: any[]): void
log
(`${
var Math: Math
Math
.
Math.round(x: number): number
round
((100 *
done: number
done
) /
total: number
total
)}%`),
});
  • cache - keeps the downloaded weights in the browser, so the next visit loads them without the network. createOpfsCache(name) is the built-in one.
  • onProgress - called while the weights upload to the GPU, with the tensor name, bytes done and bytes total. Use it for a progress bar.
  • signal - an AbortSignal. Abort it to stop the load between steps. A step already running, like the download or the GPU upload, finishes first.
ModelConfigWeightsClassesInput
MobileNetV4-Conv-Smodels.imageClassification.MOBILENETV4f16, 10.3 MBImageNet-1k224×224