Class IncrementalEncoder

constructor

• new IncrementalEncoder(
      tok: RawTokenizer,
      prefixOnlySpecials: boolean,
  ): IncrementalEncoder

  Parameters

  • tok: RawTokenizer
  • prefixOnlySpecials: boolean

  Returns IncrementalEncoder

Readonlystats

encode

• encode(text: string, special: boolean): number[]

  Parameters

  • text: string
  • special: boolean

  Returns number[]

seed

• seed(text: string, ids: number[], addSpecialTokens: boolean): void

  Test hook: inject a memo entry as if ids were the encode of text.

  Parameters

  • text: string
  • ids: number[]
  • addSpecialTokens: boolean

  Returns void