import { Tokenizer } from "https://deno.land/x/netsaur@0.3.2-patch/tokenizers/mod.ts";
Tokenizer class
Methods
decode(ids: Uint32Array, skipSpecialTokens?): string
Decode a sentence from its encoded tokens to a string
import { Tokenizer } from "https://deno.land/x/netsaur@0.3.2-patch/tokenizers/mod.ts";
Tokenizer class
Decode a sentence from its encoded tokens to a string