Object detection

Object detection finds bounding boxes for objects present in an image. You pass in the pixels and get back a list: what each object is, how sure the model is, and where it sits in the image.

createObjectDetector runs YOLO26 by Ultralytics.

Quick start

import { 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector, 
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

// 1. Load the model once. The weights download from the Hugging Face Hub.
const 
const detector: ObjectDetector
detector = await 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
objectDetection: {
    readonly YOLO26: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
objectDetection.
type YOLO26: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26.
type DEFAULT: {
    modelPath: string;
}
DEFAULT);

// 2. Get the pixels of an image, here from a canvas the picture is drawn on.
const 
const ctx: CanvasRenderingContext2D
ctx = 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.getContext(contextId: "2d", options?: CanvasRenderingContext2DSettings): CanvasRenderingContext2D | null (+5 overloads)
getContext('2d')!;
const 
const image: ImageBuffer
image = 
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData(
const ctx: CanvasRenderingContext2D
ctx.
CanvasImageData.getImageData(sx: number, sy: number, sw: number, sh: number, settings?: ImageDataSettings): ImageData
getImageData(0, 0, 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.width: number
width, 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.height: number
height));

// 3. Find the objects.
const 
const objects: ObjectDetection[]
objects = await 
const detector: ObjectDetector
detector.
ObjectDetector.detectObjects(image: ImageBuffer, options?: DetectObjectsOptions): Promise<ObjectDetection[]>
detectObjects(
const image: ImageBuffer
image);

// 4. Use them: one entry per object, best first.
for (const { 
const label: string
label, 
const confidence: number
confidence, 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box } of 
const objects: ObjectDetection[]
objects) {
  
var console: Console
console.
Console.log(...data: any[]): void
log(
const label: string
label, 
const confidence: number
confidence, 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box);
}
// bus 0.93 { format: 'xyxy', xmin: 6, ymin: 228, xmax: 807, ymax: 749 }
// person 0.92 { format: 'xyxy', xmin: 47, ymin: 399, xmax: 237, ymax: 902 }
// ...

// 5. Free the GPU memory when done.
const detector: ObjectDetector
detector.
ObjectDetector.dispose(): void
dispose();

The example reads one picture off a canvas. A video or a camera works the same way: draw each frame on the canvas and call detectObjects again, the detector is loaded once.

The engine must be set up first, see Getting started. The model runs in half precision, so the device needs shader-f16.

Try it yourself

Your camera, frame by frame through detectObjects() on your GPU, each object boxed and labeled. No camera? The sample picture goes through the same call.

Results

detectObjects returns a list with one entry per object found, best first. Nothing found gives an empty list. Each entry is:

interface ObjectDetection {
  label: string; // what it is: 'person', 'bus', ... one of the 80 COCO classes
  classId: number; // the same as a number, index into detector.labels
  confidence: number; // how sure the model is, 0 to 1
  box: { format: 'xyxy'; xmin: number; ymin: number; xmax: number; ymax: number };
}

box is the top-left corner (xmin, ymin) and the bottom-right corner (xmax, ymax) of the object, in pixels of the image you passed in.

Methods

function createObjectDetector(
  model?: ObjectDetectorModel,
  options?: LoadOptions,
): Promise<ObjectDetector>;

interface ObjectDetector {
  readonly labels: readonly string[];
  detectObjects(image: ImageBuffer, options?: DetectObjectsOptions): Promise<ObjectDetection[]>;
  dispose(): void;
}

interface DetectObjectsOptions {
  readonly confidenceThreshold?: number; // default 0.3
  readonly maxDetections?: number; // default 300
}

detectObjects(image, options?) returns the objects in the image, best first. image is an ImageBuffer of any size. Build one with imageBufferFromImageData(imageData) from a canvas, or imageBuffer(bytes, width, height, format?) from raw rgba, rgb, bgra, bgr or gray bytes. confidenceThreshold drops objects scored below it, maxDetections caps how many come back.
labels are the class names, by class index. COCO’s 80 for a standard checkpoint, class 0, class 1, … for a custom one.
dispose() frees the GPU memory.

YOLO26 picks one box per object on its own, so there is no IoU threshold.

Options

The first argument is the model config, a plain object with the fields below. models.objectDetection.YOLO26 holds ready-made ones: N (5 MB, the DEFAULT), S (18 MB) and M (39 MB, more accurate, slower). Pass one as is, copy it with a field changed, or point at your own safetensors export of an Ultralytics YOLO26 detect checkpoint.

interface ObjectDetectorModel {
  readonly modelPath?: ModelPath;
  readonly variant?: 'n' | 's' | 'm' | 'l' | 'x';
  readonly inputSize?: number; // default 640
  readonly resizeMode?: 'letterbox' | 'stretch' | 'crop'; // default 'letterbox'
}

type ModelPath = string | RangeSource; // a URL, or bytes you already have

Examples:

import { 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

// The medium size.
await 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
objectDetection: {
    readonly YOLO26: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
objectDetection.
type YOLO26: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26.
type M: {
    modelPath: string;
}
M);

// Faster, for a camera loop.
await 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector({
  ...
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
objectDetection: {
    readonly YOLO26: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
objectDetection.
type YOLO26: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26.
type DEFAULT: {
    modelPath: string;
}
DEFAULT,
  
ObjectDetectorModel.inputSize?: number | undefined
inputSize: 512,
  
ObjectDetectorModel.resizeMode?: ResizeMode | undefined
resizeMode: 'crop',
});

// Your own copy of the file.
await 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector({ 
ObjectDetectorModel.modelPath?: ModelPath | undefined
modelPath: '/models/yolo26n/model.safetensors' });

modelPath - the weights, a model.safetensors file. Default: models.objectDetection.YOLO26.DEFAULT.modelPath.
variant - the model size, n, s, m, l or x. Read from the weights by default.
inputSize - model input width and height, a multiple of 32. Default 640. Smaller is faster and misses more small objects.
resizeMode - how the image fits the model input. letterbox (default) pads, stretch distorts, crop cuts the centered square.

Loading options

The second argument controls the download. Every field is optional.

interface LoadOptions {
  cache?: WeightCache;
  cacheId?: string;
  onProgress?: (name: string, doneBytes: number, totalBytes: number) => void;
  onBytes?: (chunkBytes: number) => void;
  signal?: AbortSignal;
}

Example:

import { 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector, 
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

const 
const detector: ObjectDetector
detector = await 
function createObjectDetector(config?: ObjectDetectorModel, opts?: LoadOptions): Promise<ObjectDetector>
createObjectDetector(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
objectDetection: {
    readonly YOLO26: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
objectDetection.
type YOLO26: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26.
type DEFAULT: {
    modelPath: string;
}
DEFAULT, {
  
LoadOptions.cache?: WeightCache | undefined
cache: await 
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache('my-app'),
  
LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress: (
name: string
name, 
done: number
done, 
total: number
total) => 
var console: Console
console.
Console.log(...data: any[]): void
log(`${
var Math: Math
Math.
Math.round(x: number): number
round((100 * 
done: number
done) / 
total: number
total)}%`),
});

cache - keeps the downloaded weights in the browser, so the next visit loads them without the network. createOpfsCache(name) is the built-in one.
onProgress - called while the weights upload to the GPU, with the tensor name, bytes done and bytes total. Use it for a progress bar.
signal - an AbortSignal. Abort it to stop the load between steps. A step already running, like the download or the GPU upload, finishes first.

Supported models

Model	Config	Classes	Input	Weights
YOLO26 n	`models.objectDetection.YOLO26.N`	COCO-80	640×640 (default)	f16, 5 MB
YOLO26 s	`models.objectDetection.YOLO26.S`	COCO-80	640×640 (default)	f16, 18 MB
YOLO26 m	`models.objectDetection.YOLO26.M`	COCO-80	640×640 (default)	f16, 39 MB

Your own export of the other sizes, s, l and x, loads through modelPath too.