SplitTokenizer | /packages/utilities/src/text/preprocess/tokenize/split.ts | netsaur@0.4.0-patch

class SplitTokenizer

import { SplitTokenizer } from "https://deno.land/x/netsaur@0.4.0-patch/packages/utilities/src/text/preprocess/tokenize/split.ts";

Tokenize text based on separator (whitespace)

Constructors

new

SplitTokenizer(options?: Partial<BaseTokenizerOptions & { indices: boolean; }>)

readonly

lastToken: number

skipWords: "english" | false | string[]

Words to ignore from vocabulary

vocabulary: Map<string, number>

Configuration / Function for preprocessing

fit(text: string | string[]): this

Construct a vocabulary from a given set of text.

split(text: string): string[]

transform(text: string | string[]): number[][]

Convert a document (string | array of strings) into vectors.