vllm.entrypoints.generate.label_reads

Reads that return the logprobs of chosen label tokens at chosen positions.

Classes:

• LabelRead –

Functions:

• next_token_label_reads –

  Runs one read per input at once, as request {request_id}-{i}. Each

LabelRead dataclass

Attributes:

• logprobs (list[float]) –

  Full-vocabulary logprob of each label token, in logprob_token_ids

Source code in vllm/entrypoints/generate/label_reads.py

@dataclass(frozen=True)
class LabelRead:
    result: RequestOutput
    logprobs: list[float]
    """Full-vocabulary logprob of each label token, in ``logprob_token_ids``
    order, or slot after slot with ``slots``."""

logprobs instance-attribute

Full-vocabulary logprob of each label token, in logprob_token_ids order, or slot after slot with slots.

next_token_label_reads(engine_client, engine_inputs, sampling_params, request_id, *, lora_request=None, trace_headers=None, priority=0, slots=None) async

Runs one read per input at once, as request {request_id}-{i}. Each read's params set max_tokens=1 and the label tokens as logprob_token_ids. With slots, read i takes the logprobs of each (position, label ids) in slots[i] instead. Raises GenerationError naming the item when a read has no output or lacks a label's logprob.

Source code in vllm/entrypoints/generate/label_reads.py

async def next_token_label_reads(
    engine_client: EngineClient,
    engine_inputs: Sequence[EngineInput],
    sampling_params: Sequence[SamplingParams],
    request_id: str,
    *,
    lora_request: LoRARequest | None = None,
    trace_headers: Mapping[str, str] | None = None,
    priority: int = 0,
    slots: Sequence[Sequence[tuple[int, Sequence[int]]]] | None = None,
) -> list[LabelRead]:
    """Runs one read per input at once, as request ``{request_id}-{i}``. Each
    read's params set ``max_tokens=1`` and the label tokens as
    ``logprob_token_ids``. With ``slots``, read ``i`` takes the logprobs of each
    (position, label ids) in ``slots[i]`` instead. Raises GenerationError naming
    the item when a read has no output or lacks a label's logprob."""
    generators: list[AsyncGenerator[RequestOutput, None]] = [
        engine_client.generate(
            engine_input,
            params,
            f"{request_id}-{i}",
            lora_request=lora_request,
            trace_headers=trace_headers,
            priority=priority,
        )
        for i, (engine_input, params) in enumerate(zip(engine_inputs, sampling_params))
    ]
    results: list[RequestOutput | None] = [None] * len(generators)
    async for i, res in merge_async_iterators(*generators):
        results[i] = res

    reads = []
    for i, (result, params) in enumerate(zip(results, sampling_params)):
        if result is None:
            raise GenerationError(f"Failed to generate result for item {i}")
        if not result.outputs:
            raise GenerationError(f"No output generated for item {i}")
        output = result.outputs[0]
        if output.finish_reason == "error":
            raise GenerationError(f"Generation error for item {i}")
        if not output.logprobs:
            raise GenerationError(
                f"No logprobs available for item {i}. "
                "This might indicate an issue with logprobs configuration."
            )
        item_slots = slots[i] if slots else [(0, params.logprob_token_ids or [])]
        label_logprobs: list[float] = []
        for position, label_ids in item_slots:
            logprobs = output.logprobs[position]
            missing = [t for t in label_ids if t not in logprobs]
            if missing:
                raise GenerationError(
                    f"Token IDs {missing} not found in logprobs for item {i}. "
                    "This might indicate the tokens are outside the model's vocabulary."
                )
            label_logprobs += [logprobs[t].logprob for t in label_ids]
        reads.append(LabelRead(result, label_logprobs))
    return reads