Instance segmentation

Instance segmentation finds the objects in an image and which pixels belong to each one. You pass in the pixels and get back a list: what each object is, how sure the model is, where it sits, and a mask of its shape.

createInstanceSegmenter runs YOLO26 segment by Ultralytics on the GPU.

Quick start

import { 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter, 
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

// 1. Load the model once. The weights download from the Hugging Face Hub.
const 
const segmenter: InstanceSegmenter
segmenter = await 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
instanceSegmentation: {
    readonly YOLO26_SEG: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
instanceSegmentation.
type YOLO26_SEG: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26_SEG.
type DEFAULT: {
    modelPath: string;
}
DEFAULT);

// 2. Get the pixels of an image, here from a canvas the picture is drawn on.
const 
const ctx: CanvasRenderingContext2D
ctx = 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.getContext(contextId: "2d", options?: CanvasRenderingContext2DSettings): CanvasRenderingContext2D | null (+5 overloads)
getContext('2d')!;
const 
const image: ImageBuffer
image = 
function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData(
const ctx: CanvasRenderingContext2D
ctx.
CanvasImageData.getImageData(sx: number, sy: number, sw: number, sh: number, settings?: ImageDataSettings): ImageData
getImageData(0, 0, 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.width: number
width, 
const canvas: HTMLCanvasElement
canvas.
HTMLCanvasElement.height: number
height));

// 3. Find the objects.
const 
const objects: InstanceSegmentation[]
objects = await 
const segmenter: InstanceSegmenter
segmenter.
InstanceSegmenter.segmentInstances(image: ImageBuffer, options?: SegmentInstancesOptions): Promise<InstanceSegmentation[]>
segmentInstances(
const image: ImageBuffer
image);

// 4. Use the results.
for (const { 
const label: string
label, 
const confidence: number
confidence, 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box, 
const mask: ImageBuffer
mask } of 
const objects: InstanceSegmentation[]
objects) {
  
var console: Console
console.
Console.log(...data: any[]): void
log(
const label: string
label, 
const confidence: number
confidence, 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box, 
const mask: ImageBuffer
mask.
ImageBuffer.width: number
width, 
const mask: ImageBuffer
mask.
ImageBuffer.height: number
height);
}
// bus 0.92 { format: 'xyxy', xmin: 27, ymin: 229, xmax: 803, ymax: 726 } 115 74
// person 0.89 { format: 'xyxy', xmin: 220, ymin: 406, xmax: 345, ymax: 857 } 19 67
// ...

// 5. Free the GPU memory when done.
const segmenter: InstanceSegmenter
segmenter.
InstanceSegmenter.dispose(): void
dispose();

The example reads one picture off a canvas. A video or a camera works the same way: draw each frame on the canvas and call segmentInstances again, the segmenter is loaded once.

The engine must be set up first, see Getting started. The model runs in half precision, so the device needs shader-f16.

Try it yourself

Your camera, frame by frame through segmentInstances() on your GPU. Each mask is stretched over its object’s box in one drawImage call. No camera? The sample picture goes through the same call.

Results

segmentInstances returns a list with one entry per object found, best first. Nothing found gives an empty list. Each entry is:

interface InstanceSegmentation {
  label: string; // what it is: 'person', 'bus', ... one of the 80 COCO classes
  classId: number; // the same as a number, index into segmenter.labels
  confidence: number; // how sure the model is, 0 to 1
  box: { format: 'xyxy'; xmin: number; ymin: number; xmax: number; ymax: number };
  mask: ImageBuffer; // which pixels of the box are the object
}

interface ImageBuffer {
  data: Uint8Array; // width × height bytes, row by row: 0 outside, 255 inside
  width: number;
  height: number;
  format: 'gray';
  layout: 'hwc';
}

The mask is a small gray image, about a quarter of the model input’s resolution, that covers the object’s box: stretch it over the box when drawing. Values between 0 and 255 lie along the object’s edge, so they blend cleanly.

Drawing a mask

Turn the mask into an ImageData, then draw it scaled over box:

const 
const layer: OffscreenCanvas
layer = new 
var OffscreenCanvas: new (width: number, height: number) => OffscreenCanvas
OffscreenCanvas(1, 1);
const 
const layerCtx: OffscreenCanvasRenderingContext2D
layerCtx = 
const layer: OffscreenCanvas
layer.
OffscreenCanvas.getContext(contextId: "2d", options?: any): OffscreenCanvasRenderingContext2D | null (+5 overloads)
getContext('2d')!;

for (const { 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box, 
const mask: ImageBuffer
mask } of 
const objects: InstanceSegmentation[]
objects) {
  
const layer: OffscreenCanvas
layer.
OffscreenCanvas.width: number
width = 
const mask: ImageBuffer
mask.
ImageBuffer.width: number
width;
  
const layer: OffscreenCanvas
layer.
OffscreenCanvas.height: number
height = 
const mask: ImageBuffer
mask.
ImageBuffer.height: number
height;
  const 
const pixels: ImageData
pixels = 
const layerCtx: OffscreenCanvasRenderingContext2D
layerCtx.
CanvasImageData.createImageData(sw: number, sh: number, settings?: ImageDataSettings): ImageData (+1 overload)
createImageData(
const mask: ImageBuffer
mask.
ImageBuffer.width: number
width, 
const mask: ImageBuffer
mask.
ImageBuffer.height: number
height);
  for (let 
let i: number
i = 0; 
let i: number
i < 
const mask: ImageBuffer
mask.
ImageBuffer.data: Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike>
data.
length: number
length; 
let i: number
i++) {
    
const pixels: ImageData
pixels.
ImageData.data: ImageDataArray
data[
let i: number
i * 4] = 255; // red
    
const pixels: ImageData
pixels.
ImageData.data: ImageDataArray
data[
let i: number
i * 4 + 3] = 
const mask: ImageBuffer
mask.
ImageBuffer.data: Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike>
data[
let i: number
i] >> 1; // half transparent inside
  }
  
const layerCtx: OffscreenCanvasRenderingContext2D
layerCtx.
CanvasImageData.putImageData(imageData: ImageData, dx: number, dy: number): void (+1 overload)
putImageData(
const pixels: ImageData
pixels, 0, 0);
  const { 
const xmin: number
xmin, 
const ymin: number
ymin, 
const xmax: number
xmax, 
const ymax: number
ymax } = 
const box: {
    readonly format: "xyxy";
} & Readonly<{
    xmin: number;
    ymin: number;
    xmax: number;
    ymax: number;
}>
box;
  
const ctx: CanvasRenderingContext2D
ctx.
CanvasDrawImage.drawImage(image: CanvasImageSource, dx: number, dy: number, dw: number, dh: number): void (+2 overloads)
drawImage(
const layer: OffscreenCanvas
layer, 
const xmin: number
xmin, 
const ymin: number
ymin, 
const xmax: number
xmax - 
const xmin: number
xmin, 
const ymax: number
ymax - 
const ymin: number
ymin);
}

Methods

function createInstanceSegmenter(
  model?: InstanceSegmenterModel,
  options?: LoadOptions,
): Promise<InstanceSegmenter>;

interface InstanceSegmenter {
  readonly labels: readonly string[];
  segmentInstances(
    image: ImageBuffer,
    options?: SegmentInstancesOptions,
  ): Promise<InstanceSegmentation[]>;
  dispose(): void;
}

interface SegmentInstancesOptions {
  readonly confidenceThreshold?: number; // default 0.3
  readonly maxDetections?: number; // default 300
}

segmentInstances(image, options?) returns the objects in the image, best first, each with its mask. image is an ImageBuffer of any size. Build one with imageBufferFromImageData(imageData) from a canvas, or imageBuffer(bytes, width, height, format?) from raw rgba, rgb, bgra, bgr or gray bytes. confidenceThreshold drops objects scored below it, maxDetections caps how many come back.
labels are the class names, by class index. COCO’s 80 for a standard checkpoint, class 0, class 1, … for a custom one.
dispose() frees the GPU memory.

YOLO26 picks one box per object on its own, so there is no IoU threshold.

Options

The first argument is the model config, a plain object with the fields below. models.instanceSegmentation.YOLO26_SEG holds ready-made ones: N (5 MB, the DEFAULT), S (20 MB) and M (45 MB, more accurate, slower). Pass one as is, copy it with a field changed, or point at your own safetensors export of an Ultralytics YOLO26 segment checkpoint.

interface InstanceSegmenterModel {
  readonly modelPath?: ModelPath;
  readonly variant?: 'n' | 's' | 'm' | 'l' | 'x';
  readonly inputSize?: number; // default 640
  readonly resizeMode?: 'letterbox' | 'stretch' | 'crop'; // default 'letterbox'
}

type ModelPath = string | RangeSource; // a URL, or bytes you already have

Examples:

import { 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

// The medium size.
await 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
instanceSegmentation: {
    readonly YOLO26_SEG: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
instanceSegmentation.
type YOLO26_SEG: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26_SEG.
type M: {
    modelPath: string;
}
M);

// Faster, for a camera loop.
await 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter({
  ...
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
instanceSegmentation: {
    readonly YOLO26_SEG: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
instanceSegmentation.
type YOLO26_SEG: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26_SEG.
type DEFAULT: {
    modelPath: string;
}
DEFAULT,
  
InstanceSegmenterModel.inputSize?: number | undefined
inputSize: 512,
  
InstanceSegmenterModel.resizeMode?: ResizeMode | undefined
resizeMode: 'crop',
});

// Your own copy of the file.
await 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter({ 
InstanceSegmenterModel.modelPath?: ModelPath | undefined
modelPath: '/models/yolo26n-seg/model.safetensors' });

modelPath - the weights, a model.safetensors file. Default: models.instanceSegmentation.YOLO26_SEG.DEFAULT.modelPath.
variant - the model size, n, s, m, l or x. Read from the weights by default.
inputSize - model input width and height, a multiple of 32. Default 640. Smaller is faster, misses more small objects, and gives coarser masks.
resizeMode - how the image fits the model input. letterbox (default) pads, stretch distorts, crop cuts the centered square.

Loading options

The second argument controls the download. Every field is optional.

interface LoadOptions {
  cache?: WeightCache;
  cacheId?: string;
  onProgress?: (name: string, doneBytes: number, totalBytes: number) => void;
  onBytes?: (chunkBytes: number) => void;
  signal?: AbortSignal;
}

Example:

import { 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter, 
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache, 
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models } from 'runntime/zoo';

const 
const segmenter: InstanceSegmenter
segmenter = await 
function createInstanceSegmenter(config?: InstanceSegmenterModel, opts?: LoadOptions): Promise<InstanceSegmenter>
createInstanceSegmenter(
const models: {
    readonly embedding: {
        readonly EMBEDDINGGEMMA_2: {
            DEFAULT: {
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F16: {
                dtype: "f16";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
            F32: {
                dtype: "f32";
                modelPath: string;
                visionPath: string;
                audioPath: string;
                tokenizerPath: string;
            };
        };
        readonly ALL_MINILM_L6_V2: {
            DEFAULT: {
                tokenizerPath: string;
            } & {
                modelPath: string;
            };
            F16: {
                tokenizerPath: string;
            } & {
                dtype: "f16";
                modelPath: string;
            };
            F32: {
                tokenizerPath: string;
            } & {
                modelPath: string;
                dtype: "f32";
            };
        };
    };
    ... 6 more ...;
    readonly imageClassification: {
        readonly MOBILENETV4: {
            readonly DEFAULT: {
                modelPath: string;
            };
            readonly SMALL: {
                modelPath: string;
            };
        };
    };
}
models.
instanceSegmentation: {
    readonly YOLO26_SEG: {
        readonly DEFAULT: {
            modelPath: string;
        };
        readonly N: {
            modelPath: string;
        };
        readonly S: {
            modelPath: string;
        };
        readonly M: {
            modelPath: string;
        };
    };
}
instanceSegmentation.
type YOLO26_SEG: {
    readonly DEFAULT: {
        modelPath: string;
    };
    readonly N: {
        modelPath: string;
    };
    readonly S: {
        modelPath: string;
    };
    readonly M: {
        modelPath: string;
    };
}
YOLO26_SEG.
type DEFAULT: {
    modelPath: string;
}
DEFAULT, {
  
LoadOptions.cache?: WeightCache | undefined
cache: await 
function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache('my-app'),
  
LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress: (
name: string
name, 
done: number
done, 
total: number
total) => 
var console: Console
console.
Console.log(...data: any[]): void
log(`${
var Math: Math
Math.
Math.round(x: number): number
round((100 * 
done: number
done) / 
total: number
total)}%`),
});

cache - keeps the downloaded weights in the browser, so the next visit loads them without the network. createOpfsCache(name) is the built-in one.
onProgress - called while the weights upload to the GPU, with bytes done and bytes total. Use it for a progress bar.
signal - an AbortSignal. Abort it to cancel a load in progress.

Supported models

Model	Config	Classes	Input	Weights
YOLO26 segment n	`models.instanceSegmentation.YOLO26_SEG.N`	COCO-80	640×640 (default)	f16, 5 MB
YOLO26 segment s	`models.instanceSegmentation.YOLO26_SEG.S`	COCO-80	640×640 (default)	f16, 20 MB
YOLO26 segment m	`models.instanceSegmentation.YOLO26_SEG.M`	COCO-80	640×640 (default)	f16, 45 MB

Your own export of the other sizes, s, l and x, loads through modelPath too.