import { SplitTokenizer } from "https://deno.land/x/vectorizer@v0.7.5/src/text/preprocess/tokenize/split.ts";
Tokenize text based on separator (whitespace)
Constructors
new
SplitTokenizer(options?: Partial<BaseTokenizerOptions & { indices: boolean; }>)