import { Tokenizer } from "https://deno.land/std@0.174.0/datetime/_common.ts";
Constructors
new
Tokenizer(rules?: Rule[])Properties
rules: Rule[]
Methods
addRule(test: TestFunction, fn: CallbackFunction): Tokenizer
tokenize(string: string, receiver?): ReceiverResult[]