Tokenizer | /packages/tokenizers/mod.ts | netsaur@0.4.2

class Tokenizer

import { Tokenizer } from "https://deno.land/x/netsaur@0.4.2/packages/tokenizers/mod.ts";

Tokenizer class

Constructors

new

Tokenizer(id: number)

decode(ids: Uint32Array, skipSpecialTokens?): string

Decode a sentence from its encoded tokens to a string

encode(sentence: string): Uint32Array

Encode a sentence to tokens

getVocab(withAddedTokens?): any

Get the vocab

getVocabSize(withAddedTokens?): number

Get the vocab size

idToToken(id: number): string

Get the token from an id

save(): string

Save the tokenizer as json

save(pretty: boolean): string

Save the tokenizer as json

tokenToId(token: string): number

Get the id from a token

Load a tokenizer from json data