Variable: SPECIAL_TOKENS

const SPECIAL_TOKENS: object

Defined in: types/llm.ts:386

Special tokens used in Large Language Models (LLMs).

Type Declaration

BOS_TOKEN

BOS_TOKEN: string = 'bos_token'

CLS_TOKEN

CLS_TOKEN: string = 'cls_token'

EOS_TOKEN

EOS_TOKEN: string = 'eos_token'

MASK_TOKEN

MASK_TOKEN: string = 'mask_token'

PAD_TOKEN

PAD_TOKEN: string = 'pad_token'

SEP_TOKEN

SEP_TOKEN: string = 'sep_token'

UNK_TOKEN

UNK_TOKEN: string = 'unk_token'