Interface RequestWindow

interface RequestWindow {
    cachedPromptShare: number | null;
    count: number;
    decodeTokensPerSecond: Distribution | null;
    prefillTokensPerSecond: Distribution | null;
    queueMs: Distribution | null;
    totalMs: Distribution | null;
    ttftMs: Distribution | null;
}

Index

Properties

cachedPromptShare count decodeTokensPerSecond prefillTokensPerSecond queueMs totalMs ttftMs

Properties

`Readonly`cachedPromptShare

cachedPromptShare: number | null

Prompt tokens served from the cache, over all prompt tokens in the window; null with no prompts.

`Readonly`count

count: number

Requests the distributions cover: the most recent finished, newest last.

`Readonly`decodeTokensPerSecond

decodeTokensPerSecond: Distribution | null

`Readonly`prefillTokensPerSecond

prefillTokensPerSecond: Distribution | null

`Readonly`queueMs

queueMs: Distribution | null

`Readonly`totalMs

totalMs: Distribution | null

`Readonly`ttftMs

ttftMs: Distribution | null