Skip to content

vllm.model_executor.models.qianfan_ocr

Classes:

QianfanOCRForConditionalGeneration

Bases: InternVLChatModel

QianfanOCR multimodal model.

Identical in structure to InternVLChatModel (InternViT vision encoder + pixel-shuffle MLP connector + Qwen3 language model). This class exists solely to register the QianfanOCRForConditionalGeneration architecture name that appears in the model's config.json.

Source code in vllm/model_executor/models/qianfan_ocr.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
@MULTIMODAL_REGISTRY.register_processor(
    BaseInternVLMultiModalProcessor,
    info=QianfanOCRProcessingInfo,
    dummy_inputs=BaseInternVLDummyInputsBuilder,
)
class QianfanOCRForConditionalGeneration(InternVLChatModel):
    """QianfanOCR multimodal model.

    Identical in structure to InternVLChatModel (InternViT vision encoder +
    pixel-shuffle MLP connector + Qwen3 language model).  This class exists
    solely to register the ``QianfanOCRForConditionalGeneration`` architecture
    name that appears in the model's config.json.
    """

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
        _add_intern_vit_fields(vllm_config.model_config.hf_config)
        super().__init__(vllm_config=vllm_config, prefix=prefix)

    def _patch_quant_config(
        self, config: PreTrainedConfig, quant_config: QuantizationConfig | None
    ) -> None:
        super()._patch_quant_config(config, quant_config)
        # ignore vit layers to preserve model performance
        if isinstance(quant_config, Fp8Config) or (
            isinstance(quant_config, OnlineQuantizationConfig)
            and quant_config.args.linear is not None
            and quant_config.args.linear.weight
            in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)
        ):
            _FP8_IGNORED_LAYERS = [
                *(
                    layer
                    for i in range(config.vision_config.num_hidden_layers)
                    for layer in [
                        f"vision_model.encoder.layers.{i}.attn.qkv",
                        f"vision_model.encoder.layers.{i}.attn.proj",
                        f"vision_model.encoder.layers.{i}.mlp.fc1",
                        f"vision_model.encoder.layers.{i}.mlp.fc2",
                    ]
                ),
                "language_model.lm_head",
                "mlp1.1",
                "mlp1.3",
            ]
            quant_config.ignored_layers = list(quant_config.ignored_layers)
            for layer in _FP8_IGNORED_LAYERS:
                if layer not in quant_config.ignored_layers:
                    quant_config.ignored_layers.append(layer)

QianfanOCRProcessingInfo

Bases: BaseInternVLProcessingInfo

Image-only ProcessingInfo for QianfanOCR (no video support).

Source code in vllm/model_executor/models/qianfan_ocr.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
    """Image-only ProcessingInfo for QianfanOCR (no video support)."""

    def get_hf_config(self) -> QianfanOCRConfig:
        return _add_intern_vit_fields(self.ctx.get_hf_config(QianfanOCRConfig))

    def get_hf_processor(self, **kwargs: object) -> InternVLChatProcessor:
        config = self.get_hf_config()
        vision_config = config.vision_config

        merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
        merged_kwargs.setdefault("image_size", vision_config.image_size)
        merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
        merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
        merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
        merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail)

        image_processor = InternVLChatImageProcessor(**merged_kwargs)
        image_size = image_processor.image_size
        patch_size = vision_config.patch_size
        downsample_ratio = config.downsample_ratio
        image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))

        return InternVLChatProcessor(
            tokenizer=self.get_tokenizer(),
            image_processor=image_processor,
            video_processor=None,
            image_seq_length=image_seq_length,
            ctx_video_token=None,
        )

_add_intern_vit_fields(config)

Expose the upstream vision config under the field names InternViT reads.

Source code in vllm/model_executor/models/qianfan_ocr.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
def _add_intern_vit_fields(config: QianfanOCRConfig) -> QianfanOCRConfig:
    """Expose the upstream vision config under the field names InternViT reads."""
    vision_config = config.vision_config
    for name in ("image_size", "patch_size"):
        if isinstance(size := getattr(vision_config, name), (list, tuple)):
            setattr(vision_config, name, size[0])
    if not hasattr(vision_config, "qkv_bias"):
        vision_config.qkv_bias = vision_config.attention_bias
    if not hasattr(vision_config, "qk_normalization"):
        vision_config.qk_normalization = vision_config.use_qk_norm
    if not hasattr(vision_config, "initializer_factor"):
        vision_config.initializer_factor = vision_config.layer_scale_init_value
    return config