vllm.entrypoints.openai.decisions.serving

Classes:

• BaseServingDecisions –

BaseServingDecisions

Bases: BaseServing

Source code in vllm/entrypoints/openai/decisions/serving.py

class BaseServingDecisions(BaseServing):
    def __init__(
        self,
        models: OpenAIServingModels,
        strategy: ReadStrategy,
        *,
        request_logger: RequestLogger | None = None,
    ) -> None:
        super().__init__(
            models=models,
            model_config=strategy.context.engine_client.model_config,
            request_logger=request_logger,
        )
        self.strategy = strategy
        self.limits = strategy.limits()

    def _admit(self, questions: list[Question], joint: bool) -> bool:
        """Admits the reads and returns whether they read jointly."""
        if joint and not self.strategy.joint_reads:
            raise StructuredDecisionError(
                "joint: this model reads each question on its own; "
                "joint reads need DiffusionGemma"
            )
        # One question reads best with the per-question prompt.
        joint = joint and len(questions) > 1
        self.strategy.context.engine_client.check_admission(
            self.strategy.num_requests(questions, joint=joint)
        )
        return joint

_admit(questions, joint)

Admits the reads and returns whether they read jointly.

Source code in vllm/entrypoints/openai/decisions/serving.py

def _admit(self, questions: list[Question], joint: bool) -> bool:
    """Admits the reads and returns whether they read jointly."""
    if joint and not self.strategy.joint_reads:
        raise StructuredDecisionError(
            "joint: this model reads each question on its own; "
            "joint reads need DiffusionGemma"
        )
    # One question reads best with the per-question prompt.
    joint = joint and len(questions) > 1
    self.strategy.context.engine_client.check_admission(
        self.strategy.num_requests(questions, joint=joint)
    )
    return joint