import { type BaseTokenizerOptions } from "https://deno.land/x/vectorizer@v0.7.5/src/utils/common_types.ts";
definition: { vocabulary: Map<string, number>; standardize: StandardizeConfig | ((s: string) => string); skipWords: "english" | false | string[]; }