@lancedb/lancedb • Docs

---

@lancedb/lancedb / BaseTokenizer

Type Alias: BaseTokenizer

type BaseTokenizer:
  | "simple"
  | "whitespace"
  | "raw"
  | "ngram"
  | "icu"
  | "icu/split"
  | `jieba/${string}`
  | `lindera/${string}`;