Pose & keypoints
Keypoint detection finds the people in an image and where their body landmarks are. You pass in the pixels and get back a list: one entry per person, with a box and 17 landmarks, from the nose to the ankles.
createKeypointDetector runs
YOLO26 pose by Ultralytics on
the GPU.
Quick start
Section titled “Quick start”import { function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector, function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';
// 1. Load the model once. The weights download from the Hugging Face Hub.const const detector: KeypointDetector
detector = await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector(const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.keypointDetection: { readonly YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; }; };}
keypointDetection.type YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; };}
YOLO26_POSE.type DEFAULT: { modelPath: string;}
DEFAULT);
// 2. Get the pixels of an image, here from a canvas the picture is drawn on.const const ctx: CanvasRenderingContext2D
ctx = const canvas: HTMLCanvasElement
canvas.HTMLCanvasElement.getContext(contextId: "2d", options?: CanvasRenderingContext2DSettings): CanvasRenderingContext2D | null (+5 overloads)
getContext('2d')!;const const image: ImageBuffer
image = function imageBufferFromImageData(img: ImageData): ImageBuffer
imageBufferFromImageData(const ctx: CanvasRenderingContext2D
ctx.CanvasImageData.getImageData(sx: number, sy: number, sw: number, sh: number, settings?: ImageDataSettings): ImageData
getImageData(0, 0, const canvas: HTMLCanvasElement
canvas.HTMLCanvasElement.width: number
width, const canvas: HTMLCanvasElement
canvas.HTMLCanvasElement.height: number
height));
// 3. Find the people.const const people: KeypointDetection[]
people = await const detector: KeypointDetector
detector.KeypointDetector.detectKeypoints(image: ImageBuffer, options?: DetectKeypointsOptions): Promise<KeypointDetection[]>
detectKeypoints(const image: ImageBuffer
image);
// 4. Use the results.for (const { const confidence: number
confidence, const box: { readonly format: "xyxy";} & Readonly<{ xmin: number; ymin: number; xmax: number; ymax: number;}>
box, const landmarks: Readonly<Record<"nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle", Landmark>>
landmarks } of const people: KeypointDetection[]
people) { var console: Console
console.Console.log(...data: any[]): void
log(const confidence: number
confidence, const box: { readonly format: "xyxy";} & Readonly<{ xmin: number; ymin: number; xmax: number; ymax: number;}>
box, const landmarks: Readonly<Record<"nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle", Landmark>>
landmarks.nose: Landmark
nose);}// 0.89 { format: 'xyxy', xmin: 48, ymin: 397, xmax: 241, ymax: 908 } { x: 144, y: 446, confidence: 0.99 }// 0.87 { format: 'xyxy', xmin: 224, ymin: 404, xmax: 346, ymax: 868 } { x: 296, y: 448, confidence: 0.99 }// ...
// 5. Free the GPU memory when done.const detector: KeypointDetector
detector.KeypointDetector.dispose(): void
dispose();The example reads one picture off a canvas. A video or a camera works the
same way: draw each frame on the canvas and call detectKeypoints again,
the detector is loaded once.
The engine must be set up first, see Getting started.
The model runs in half precision, so the device needs shader-f16.
Try it yourself
Section titled “Try it yourself”Your camera, frame by frame through detectKeypoints() on your GPU. Limbs
are drawn from detector.skeleton, landmarks under 0.5 confidence are left
out. No camera? The sample picture goes through the same call.
Results
Section titled “Results”detectKeypoints returns a list with one entry per person found, best
first. Nobody found gives an empty list. Each entry is:
interface KeypointDetection { box: { format: 'xyxy'; xmin: number; ymin: number; xmax: number; ymax: number }; confidence: number; // how sure the model is, 0 to 1 landmarks: Record<CocoLandmark, Landmark>; // 17 entries, 'nose' to 'rightAnkle'}
interface Landmark { x: number; // in pixels of the image you passed in y: number; confidence: number; // how sure the model is the landmark is visible, 0 to 1}Every person has all 17 landmarks. One the model cannot see, behind the
body or outside the frame, still has a position, with a low confidence.
Skip landmarks below about 0.5 when drawing.
Drawing a skeleton
Section titled “Drawing a skeleton”detector.skeleton lists which landmarks connect, as pairs of names:
for (const const person: KeypointDetection
person of const people: KeypointDetection[]
people) { for (const [const a: "nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle"
a, const b: "nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle"
b] of const detector: KeypointDetector
detector.KeypointDetector.skeleton: readonly (readonly ["nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle", "nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle"])[]
skeleton) { const const from: Landmark
from = const person: KeypointDetection
person.KeypointDetection.landmarks: Readonly<Record<"nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle", Landmark>>
landmarks[const a: "nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle"
a]; const const to: Landmark
to = const person: KeypointDetection
person.KeypointDetection.landmarks: Readonly<Record<"nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle", Landmark>>
landmarks[const b: "nose" | "leftEye" | "rightEye" | "leftEar" | "rightEar" | "leftShoulder" | "rightShoulder" | "leftElbow" | "rightElbow" | "leftWrist" | "rightWrist" | "leftHip" | "rightHip" | "leftKnee" | "rightKnee" | "leftAnkle" | "rightAnkle"
b]; if (const from: Landmark
from.Landmark.confidence: number
confidence < 0.5 || const to: Landmark
to.Landmark.confidence: number
confidence < 0.5) continue; const ctx: CanvasRenderingContext2D
ctx.CanvasPath.moveTo(x: number, y: number): void
moveTo(const from: Landmark
from.Point.x: number
x, const from: Landmark
from.Point.y: number
y); const ctx: CanvasRenderingContext2D
ctx.CanvasPath.lineTo(x: number, y: number): void
lineTo(const to: Landmark
to.Point.x: number
x, const to: Landmark
to.Point.y: number
y); }}const ctx: CanvasRenderingContext2D
ctx.CanvasDrawPath.stroke(): void (+1 overload)
stroke();Methods
Section titled “Methods”function createKeypointDetector( model?: KeypointDetectorModel, options?: LoadOptions,): Promise<KeypointDetector>;
interface KeypointDetector { readonly landmarkNames: readonly CocoLandmark[]; readonly skeleton: readonly (readonly [CocoLandmark, CocoLandmark])[]; readonly inputSize: number; readonly schemas: { readonly input: Yolo26Input; readonly output: (maxDetections: number) => KeypointDetectionList; }; detectKeypoints( image: ImageBuffer, options?: DetectKeypointsOptions, ): Promise<KeypointDetection[]>; encodeDetectKeypoints(options: EncodeDetectKeypointsOptions): void; dispose(): void;}
interface DetectKeypointsOptions { readonly confidenceThreshold?: number; // default 0.3 readonly maxDetections?: number; // default 300}
type CocoLandmark = | 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';detectKeypoints(image, options?)returns the people in the image, best first.imageis anImageBufferof any size. Build one withimageBufferFromImageData(imageData)from a canvas, orimageBuffer(bytes, width, height, format?)from rawrgba,rgb,bgra,bgrorgraybytes.confidenceThresholddrops people scored below it,maxDetectionscaps how many come back.landmarkNamesare the 17 names in model order, the same list as the exportedCOCO_LANDMARKS.skeletonare the landmark pairs that form limbs, as name pairs.encodeDetectKeypoints(options)reads an image already on the GPU and writes the people and their landmarks to a GPU buffer, see GPU in, GPU out.inputSizeis the model input width and height, 640 by default.schemasare the TypeGPU layouts of the model input and of the listencodeDetectKeypointswrites, for buffers passed to it.dispose()frees the GPU memory.
YOLO26 picks one box per person on its own, so there is no IoU threshold.
GPU in, GPU out
Section titled “GPU in, GPU out”When the image is already on the GPU, a video frame or the output of your
own shader, encodeDetectKeypoints skips the CPU: no pixels are read back, and the
people are picked and decoded on the GPU, straight into a buffer you own,
for your next pass to draw or read.
import { function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector, function initRunntime(root: TgpuRoot): RunntimeExecutor
initRunntime, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';import const tgpu: { const: { <TDataType extends AnyData>(dataType: TDataType, value: InferGPU<TDataType>): TgpuConst<TDataType>; <TElement extends AnyWgslData>(dataType: (elementCount: number) => WgslArray<TElement>, value: InferGPU<WgslArray<TElement>>): TgpuConst<WgslArray<TElement>>; }; fn: { <Args extends BaseData[] | []>(argTypes: Args, returnType?: undefined): TgpuFnShell<Args, Void>; <Args extends BaseData[] | [], Return extends BaseData>(argTypes: Args, returnType: Return): TgpuFnShell<Args, Return>; <T extends AnyFn>(inner: T): TgpuGenericFn<T>; }; ... 17 more ...; '~unstable': typeof import("/home/runner/work/runntime/runntime/node_modules/.pnpm/typegpu@0.12.5/node_modules/typegpu/tgpuUnstable");}
tgpu from 'typegpu';
// 1. Set up the engine and load the model once.const const root: TgpuRoot
root = await const tgpu: { const: { <TDataType extends AnyData>(dataType: TDataType, value: InferGPU<TDataType>): TgpuConst<TDataType>; <TElement extends AnyWgslData>(dataType: (elementCount: number) => WgslArray<TElement>, value: InferGPU<WgslArray<TElement>>): TgpuConst<WgslArray<TElement>>; }; fn: { <Args extends BaseData[] | []>(argTypes: Args, returnType?: undefined): TgpuFnShell<Args, Void>; <Args extends BaseData[] | [], Return extends BaseData>(argTypes: Args, returnType: Return): TgpuFnShell<Args, Return>; <T extends AnyFn>(inner: T): TgpuGenericFn<T>; }; ... 17 more ...; '~unstable': typeof import("/home/runner/work/runntime/runntime/node_modules/.pnpm/typegpu@0.12.5/node_modules/typegpu/tgpuUnstable");}
tgpu.init: (options?: InitOptions) => Promise<TgpuRoot>
init({ device?: (GPUDeviceDescriptor & { optionalFeatures?: Iterable<GPUFeatureName>;}) | undefined
device: { optionalFeatures?: Iterable<GPUFeatureName> | undefined
optionalFeatures: ['subgroups', 'shader-f16'] } });function initRunntime(root: TgpuRoot): RunntimeExecutor
initRunntime(const root: TgpuRoot
root);const const detector: KeypointDetector
detector = await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector(const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.keypointDetection: { readonly YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; }; };}
keypointDetection.type YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; };}
YOLO26_POSE.type DEFAULT: { modelPath: string;}
DEFAULT);
// 2. A buffer for up to 100 people, typed by the detector's schema.const const people: TgpuBuffer<KeypointDetectionList> & StorageFlag
people = const root: TgpuRoot
root.TgpuRoot.createBuffer<KeypointDetectionList>(typeSchema: KeypointDetectionList, initial?: ((buffer: TgpuBuffer<NoInfer<KeypointDetectionList>>) => void) | InferInput<NoInfer<KeypointDetectionList>> | undefined): TgpuBuffer<KeypointDetectionList> (+1 overload)
createBuffer(const detector: KeypointDetector
detector.KeypointDetector.schemas: { readonly input: Yolo26Input; readonly output: (maxDetections: number) => KeypointDetectionList;}
schemas.output: (maxDetections: number) => KeypointDetectionList
output(100)).TgpuBuffer<KeypointDetectionList>.$usage<["storage"]>(usages_0: "storage"): TgpuBuffer<KeypointDetectionList> & StorageFlag
$usage('storage');
// 3. Each frame: video in, the list out, then your own passes in the same encoder.const const encoder: TgpuCommandEncoder
encoder = const root: TgpuRoot
root['~unstable'].function createCommandEncoder(descriptor?: GPUCommandEncoderDescriptor): TgpuCommandEncoder
createCommandEncoder();const detector: KeypointDetector
detector.KeypointDetector.encodeDetectKeypoints(options: EncodeDetectKeypointsOptions): void
encodeDetectKeypoints({ EncodeOptions<GpuImage | GpuTensor<Yolo26Input>, GpuTensor<KeypointDetectionList>>.source: GpuImage | GpuTensor<Yolo26Input>
source: const root: TgpuRoot
root.TgpuRoot.device: GPUDevice
device.GPUDevice.importExternalTexture(descriptor: GPUExternalTextureDescriptor): GPUExternalTexture
importExternalTexture({ GPUExternalTextureDescriptor.source: HTMLVideoElement | VideoFrame
source: const video: HTMLVideoElement
video }), EncodeOptions<GpuImage | GpuTensor<Yolo26Input>, GpuTensor<KeypointDetectionList>>.target: GpuTensor<KeypointDetectionList>
target: const people: TgpuBuffer<KeypointDetectionList> & StorageFlag
people, EncodeDetectKeypointsOptions.confidenceThreshold?: number | undefined
confidenceThreshold: 0.5, EncodeOptions<GpuImage | GpuTensor<Yolo26Input>, GpuTensor<KeypointDetectionList>>.encoder?: GpuEncoder | undefined
encoder,});// ...record your own passes that read `people` here...const encoder: TgpuCommandEncoder
encoder.TgpuCommandEncoder.submit(): void
submit();Source - one of:
- a texture: a
GPUTexture, a TypeGPU texture withsampledusage, or aGPUExternalTexturefrom a video. It is fitted like anImageBuffer, with the detector’sresizeMode, and each value 0-1 counts as a byte 0-255. Formats:rgba8unorm,bgra8unorm,rgb10a2unorm,rgba16float,rgba32float. sRGB formats are rejected: the GPU would turn their values linear, which the model was not trained on. - a buffer holding the model input as is: 3×640×640 f32 at the default
inputSize, normalized RGB, one plane per channel. A rawGPUBufferneedsCOPY_SRCusage.
Target - a buffer with STORAGE usage, laid out as
schemas.output(maxDetections):
struct Box { xmin: f32, ymin: f32, xmax: f32, ymax: f32 }struct Landmark { x: f32, y: f32, confidence: f32 }struct KeypointDetection { box: Box, confidence: f32, landmarks: array<Landmark, 17> }struct KeypointDetectionList { count: u32, detections: array<KeypointDetection, maxDetections> }The first count entries are the people detectKeypoints would return, best
first. box is in pixels of the source image, cut at its edge; for a buffer source, of the model input. confidence is 0 to 1. landmarks are in landmarkNames order, in the same pixels, not cut at the edge. Entries past count keep old values. Every field is 4 bytes
with no padding, so in a raw GPUBuffer entry i starts at byte
4 + 224 * i.
confidenceThreshold drops people scored below it, 0.3 by default.
maxDetections caps how many are written, by default as many as the target
holds.
encoder - optional. Pass a GPUCommandEncoder or a TypeGPU command
encoder to record the detection together with your own passes into one command
buffer. You then finish and submit it yourself, before dispose(). Without
it, the call submits its own before it returns, and does not wait for the
GPU. Work you submit after it sees the result.
Every resource must come from the device passed to initRunntime(). A
wrong format, usage or size, or an option out of range, throws
INVALID_ARGUMENT.
Options
Section titled “Options”The first argument is the model config, a plain object with the fields
below. models.keypointDetection.YOLO26_POSE holds ready-made ones: N
(6 MB, the DEFAULT), S (20 MB) and M (42 MB, more accurate, slower). Pass one as
is, copy it with a field changed, or point at your own safetensors export
of an Ultralytics YOLO26 pose checkpoint.
interface KeypointDetectorModel { readonly modelPath?: ModelPath; readonly variant?: 'n' | 's' | 'm' | 'l' | 'x'; readonly inputSize?: number; // default 640 readonly resizeMode?: 'letterbox' | 'stretch' | 'crop'; // default 'letterbox'}
type ModelPath = string | RangeSource; // a URL, or bytes you already haveExamples:
import { function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';
// The medium size.await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector(const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.keypointDetection: { readonly YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; }; };}
keypointDetection.type YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; };}
YOLO26_POSE.type M: { modelPath: string;}
M);
// Faster, for a camera loop.await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector({ ...const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.keypointDetection: { readonly YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; }; };}
keypointDetection.type YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; };}
YOLO26_POSE.type DEFAULT: { modelPath: string;}
DEFAULT, KeypointDetectorModel.inputSize?: number | undefined
inputSize: 512, KeypointDetectorModel.resizeMode?: ResizeMode | undefined
resizeMode: 'crop',});
// Your own copy of the file.await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector({ KeypointDetectorModel.modelPath?: ModelPath | undefined
modelPath: '/models/yolo26n-pose/model.safetensors' });- modelPath - the weights, a
model.safetensorsfile. Default:models.keypointDetection.YOLO26_POSE.DEFAULT.modelPath. - variant - the model size,
n,s,m,lorx. Read from the weights by default. - inputSize - model input width and height, a multiple of 32. Default 640. Smaller is faster and misses more small people.
- resizeMode - how the image fits the model input.
letterbox(default) pads,stretchdistorts,cropcuts the centered square.
Loading options
Section titled “Loading options”The second argument controls the download and the warm-up run. Every field is optional.
interface LoadOptions { cache?: WeightCache; cacheId?: string; onProgress?: (name: string, doneBytes: number, totalBytes: number) => void; onBytes?: (chunkBytes: number) => void; signal?: AbortSignal; warmUp?: boolean;}By default create runs the model once on a blank input before it returns,
so the kernels compile during the load and a GPU error fails the load.
With warmUp: false it skips that run and returns sooner. The first call
then pays the compile time, and a GPU error is not reported: the call
returns zeros instead of throwing.
Example:
import { function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector, function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache, const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models } from 'runntime/zoo';
const const detector: KeypointDetector
detector = await function createKeypointDetector(config?: KeypointDetectorModel, opts?: LoadOptions): Promise<KeypointDetector>
createKeypointDetector(const models: { readonly embedding: { readonly EMBEDDINGGEMMA_2: { DEFAULT: { modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F16: { dtype: "f16"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; F32: { dtype: "f32"; modelPath: string; visionPath: string; audioPath: string; tokenizerPath: string; }; }; readonly ALL_MINILM_L6_V2: { DEFAULT: { tokenizerPath: string; } & { modelPath: string; }; F16: { tokenizerPath: string; } & { dtype: "f16"; modelPath: string; }; F32: { tokenizerPath: string; } & { modelPath: string; dtype: "f32"; }; }; }; ... 7 more ...; readonly imageClassification: { readonly MOBILENETV4: { readonly DEFAULT: { modelPath: string; }; readonly SMALL: { modelPath: string; }; }; };}
models.keypointDetection: { readonly YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; }; };}
keypointDetection.type YOLO26_POSE: { readonly DEFAULT: { modelPath: string; }; readonly N: { modelPath: string; }; readonly S: { modelPath: string; }; readonly M: { modelPath: string; };}
YOLO26_POSE.type DEFAULT: { modelPath: string;}
DEFAULT, { LoadOptions.cache?: WeightCache | undefined
cache: await function createOpfsCache(namespace: string): Promise<WeightCache | undefined>
createOpfsCache('my-app'), LoadOptions.onProgress?: ((name: string, doneBytes: number, totalBytes: number) => void) | undefined
onProgress: (name: string
name, done: number
done, total: number
total) => var console: Console
console.Console.log(...data: any[]): void
log(`${var Math: Math
Math.Math.round(x: number): number
round((100 * done: number
done) / total: number
total)}%`),});- cache - keeps the downloaded weights in the browser, so the next
visit loads them without the network.
createOpfsCache(name)is the built-in one. - onProgress - called while the weights upload to the GPU, with bytes done and bytes total. Use it for a progress bar.
- signal - an
AbortSignal. Abort it to cancel a load in progress.
Supported models
Section titled “Supported models”| Model | Config | Landmarks | Input | Weights |
|---|---|---|---|---|
| YOLO26 pose n | models.keypointDetection.YOLO26_POSE.N | COCO-17 | 640×640 (default) | f16, 6 MB |
| YOLO26 pose s | models.keypointDetection.YOLO26_POSE.S | COCO-17 | 640×640 (default) | f16, 20 MB |
| YOLO26 pose m | models.keypointDetection.YOLO26_POSE.M | COCO-17 | 640×640 (default) | f16, 42 MB |
Your own export of the other sizes, s, l and x, loads through
modelPath too.