vllm.model_executor.layers.quantization.utils.humming.moe

Configure, prepare weights for, and assemble Humming MoE kernels.

Functions:

• convert_to_humming_moe_kernel_format –

  Convert MoE weights from checkpoint format to Humming kernel format.

• select_humming_moe_experts –

  Select the primary Humming MoE Experts class

• stage_moe_weights_for_humming –

  Install canonical weights and scales for Humming's layer-based converter.

_convert_sublayer_to_humming(layer, sublayer_name, shape_n, shape_k, weight_schema, input_schema, num_experts, param_dtype)

Convert a sublayer's weights from checkpoint format to Humming format.

Returns:

• tuple[Any, Any] –

  Tuple of (converted_weight_schema, converted_input_schema)

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def _convert_sublayer_to_humming(
    layer: "RoutedExperts",
    sublayer_name: str,
    shape_n: int,
    shape_k: int,
    weight_schema: Any,
    input_schema: Any,
    num_experts: int,
    param_dtype: torch.dtype,
) -> tuple[Any, Any]:
    """Convert a sublayer's weights from checkpoint format to Humming format.

    Returns:
        Tuple of (converted_weight_schema, converted_input_schema)

    """
    tensors = _extract_sublayer_tensors(layer, sublayer_name)

    shape_k_stacks = [shape_k]
    shape_n_stacks = [shape_n]
    if sublayer_name == "w13" and layer.moe_config.activation.is_gated:
        shape_n_stacks = [shape_n // 2] * 2

    converted_weight_schema, converted_tensors = weight_schema.convert_humming(
        tensors=tensors,
        shape_n_stacks=shape_n_stacks,
        shape_k_stacks=shape_k_stacks,
        param_dtype=param_dtype,
        num_experts=num_experts,
    )

    converted_input_schema, input_tensors = input_schema.convert_humming(
        tensors=tensors,
        shape_n_stacks=shape_n_stacks,
        shape_k_stacks=shape_k_stacks,
        param_dtype=param_dtype,
        num_experts=num_experts,
    )
    converted_tensors.update(input_tensors)

    _replace_layer_parameters(layer, sublayer_name, converted_tensors)

    return converted_weight_schema, converted_input_schema

_extract_sublayer_tensors(layer, sublayer_name)

Extract a sublayer's live params and persistent buffers.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def _extract_sublayer_tensors(
    layer: "RoutedExperts",
    sublayer_name: str,
) -> dict[str, torch.Tensor]:
    """Extract a sublayer's live params and persistent buffers."""
    buffers = {
        name: buffer
        for name, buffer in layer._buffers.items()
        if buffer is not None and name not in layer._non_persistent_buffers_set
    }
    return {
        key.removeprefix(sublayer_name + "_"): value
        for key, value in (kernel_parameters(layer) | buffers).items()
        if key.startswith(sublayer_name + "_")
    }

_prepare_and_transform_sublayer(layer, sublayer_name, shape_n, shape_k, weight_schema, input_schema, has_bias, num_experts, param_dtype)

Prepare Humming configuration and transform one sublayer's tensors.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def _prepare_and_transform_sublayer(
    layer: "RoutedExperts",
    sublayer_name: str,
    shape_n: int,
    shape_k: int,
    weight_schema: Any,
    input_schema: Any,
    has_bias: bool,
    num_experts: int,
    param_dtype: torch.dtype,
) -> "LayerConfig":
    """Prepare Humming configuration and transform one sublayer's tensors."""
    from vllm.utils.humming import (
        prepare_and_transform_humming_tensors,
    )

    source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
    for name in ("input_scale", "input_scale_2"):
        if name != input_schema.static_tensor_scale_name:
            source_tensors.pop(name, None)
    config, tensors = prepare_and_transform_humming_tensors(
        tensors=source_tensors,
        shape_n=shape_n,
        shape_k=shape_k,
        pad_n_to_multiple=256,
        pad_k_to_multiple=128,
        input_schema=input_schema,
        weight_schema=weight_schema,
        has_bias=has_bias,
        num_experts=num_experts,
        torch_dtype=param_dtype,
        device=getattr(layer, f"{sublayer_name}_weight").device,
    )
    for name in ("input_scale", "input_scale_2"):
        if name in source_tensors:
            tensors[name] = source_tensors[name]
    _replace_layer_parameters(layer, sublayer_name, tensors)
    return config

_process_single_sublayer(layer, sublayer_name, shape_n, shape_k, weight_schema, input_schema, has_bias, num_experts, param_dtype, force_weight_schema=None, allow_input_schema_fallback=True)

Process a single sublayer: convert, optionally requant, prepare, and transform.

This combines the common logic from convert_to_humming_moe_kernel_format for processing a single sublayer.

Parameters:

• layer

  (RoutedExperts) –

  The RoutedExperts layer

• sublayer_name

  (str) –

  Name of the sublayer (e.g., "w13", "w2")

• shape_n

  (int) –

  Output dimension size

• shape_k

  (int) –

  Input dimension size

• weight_schema

  (Any) –

  Initial weight quantization schema

• input_schema

  (Any) –

  Initial input quantization schema

• has_bias

  (bool) –

  Whether the layer has bias terms

• num_experts

  (int) –

  Number of experts

• param_dtype

  (dtype) –

  Parameter data type

• force_weight_schema

  (Any | None, default: None ) –

  Optional schema to force requantization to

• allow_input_schema_fallback

  (bool, default: True ) –

  Whether incompatible input schemas may be replaced.

Returns:

• tuple[Any, Any, LayerConfig] –

  Tuple of the final weight schema, input schema, and Humming layer config.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def _process_single_sublayer(
    layer: "RoutedExperts",
    sublayer_name: str,
    shape_n: int,
    shape_k: int,
    weight_schema: Any,
    input_schema: Any,
    has_bias: bool,
    num_experts: int,
    param_dtype: torch.dtype,
    force_weight_schema: Any | None = None,
    allow_input_schema_fallback: bool = True,
) -> tuple[Any, Any, "LayerConfig"]:
    """Process a single sublayer: convert, optionally requant, prepare, and transform.

    This combines the common logic from convert_to_humming_moe_kernel_format
    for processing a single sublayer.

    Args:
        layer: The RoutedExperts layer
        sublayer_name: Name of the sublayer (e.g., "w13", "w2")
        shape_n: Output dimension size
        shape_k: Input dimension size
        weight_schema: Initial weight quantization schema
        input_schema: Initial input quantization schema
        has_bias: Whether the layer has bias terms
        num_experts: Number of experts
        param_dtype: Parameter data type
        force_weight_schema: Optional schema to force requantization to
        allow_input_schema_fallback: Whether incompatible input schemas may be replaced.

    Returns:
        Tuple of the final weight schema, input schema, and Humming layer config.

    """
    from vllm.utils.humming import HummingWeightSchema

    # Step 1: Convert from checkpoint format to humming format if needed
    current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
        layer=layer,
        sublayer_name=sublayer_name,
        shape_n=shape_n,
        shape_k=shape_k,
        weight_schema=weight_schema,
        input_schema=input_schema,
        num_experts=num_experts,
        param_dtype=param_dtype,
    )

    # Step 2: Force requant if needed
    assert isinstance(current_weight_schema, HummingWeightSchema)
    if force_weight_schema is not None and current_weight_schema != force_weight_schema:
        source_tensors = _extract_sublayer_tensors(layer, sublayer_name)

        tensors = current_weight_schema.requant_tensors(
            tensors=source_tensors,
            target_weight_schema=force_weight_schema,
            param_dtype=param_dtype,
        )
        name = current_input_schema.static_tensor_scale_name
        if name is not None:
            tensors[name] = source_tensors[name]

        current_weight_schema = force_weight_schema
        _replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=True)
        del tensors

    # Step 3: Prepare layer metadata and transform weights
    current_input_schema = humming_schema.check_and_fallback_input_schema(
        weight_schema=current_weight_schema,
        input_schema=current_input_schema,
        param_dtype=param_dtype,
        allow_fallback=allow_input_schema_fallback,
    )
    config = _prepare_and_transform_sublayer(
        layer=layer,
        sublayer_name=sublayer_name,
        shape_n=shape_n,
        shape_k=shape_k,
        weight_schema=current_weight_schema,
        input_schema=current_input_schema,
        has_bias=has_bias,
        num_experts=num_experts,
        param_dtype=param_dtype,
    )

    return current_weight_schema, current_input_schema, config

_replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=False)

Replace layer parameters for a sublayer with new tensors.

Parameters:

• layer

  (RoutedExperts) –

  The RoutedExperts layer

• sublayer_name

  (str) –

  Name of the sublayer (e.g., "w13", "w2")

• tensors

  (dict[str, Tensor]) –

  Dict of parameter name to tensor

• preserve_bias

  (bool, default: False ) –

  If True, don't delete bias parameters

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def _replace_layer_parameters(
    layer: "RoutedExperts",
    sublayer_name: str,
    tensors: dict[str, torch.Tensor],
    preserve_bias: bool = False,
) -> None:
    """Replace layer parameters for a sublayer with new tensors.

    Args:
        layer: The RoutedExperts layer
        sublayer_name: Name of the sublayer (e.g., "w13", "w2")
        tensors: Dict of parameter name to tensor
        preserve_bias: If True, don't delete bias parameters

    """
    names = [
        name
        for name in layer._parameters
        if name.startswith(sublayer_name + "_")
        and not (preserve_bias and name == sublayer_name + "_bias")
    ]
    replace_layer_parameters(
        layer, {f"{sublayer_name}_{n}": t for n, t in tensors.items()}, names
    )

convert_to_humming_moe_kernel_format(layer, quant_config=None, sublayer_configs=None, weight_schema=None, input_schema=None, force_weight_schema=None, allow_input_schema_fallback=True)

Convert MoE weights from checkpoint format to Humming kernel format.

This function processes weights for each sublayer (w13, w2) by: 1. Converting from checkpoint format to humming format if needed 2. Force requanting if a different quantization schema is specified 3. Preparing layer metadata for the Humming kernel 4. Transforming weights for inference

Parameters:

• layer

  (RoutedExperts) –

  The RoutedExperts layer containing weights to process

• quant_config

  (dict | None, default: None ) –

  Optional quantization config dict. Required if weight_schema or input_schema are None. Used to build schemas via BaseWeightSchema.from_config().

• sublayer_configs

  (dict[str, Any] | None, default: None ) –

  Optional configuration dict for each sublayer (w13, w2). Each config must have "shape_n" and "shape_k" keys. If None, configs are built from layer.moe_config properties.

• weight_schema

  (Any | None, default: None ) –

  Optional initial weight quantization schema. If None, built from quant_config.

• input_schema

  (Any | None, default: None ) –

  Optional initial input quantization schema. If None, built from quant_config or env vars.

• force_weight_schema

  (Any | None, default: None ) –

  Optional schema to force requantization to

• allow_input_schema_fallback

  (bool, default: True ) –

  Whether incompatible input schemas may be replaced.

Side effects

• Modifies layer parameters in place
• Sets layer.weight_schemas and layer.input_schemas
• Sets layer.humming_configs for quant config construction

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def convert_to_humming_moe_kernel_format(
    layer: "RoutedExperts",
    quant_config: dict | None = None,
    sublayer_configs: dict[str, Any] | None = None,
    weight_schema: Any | None = None,
    input_schema: Any | None = None,
    force_weight_schema: Any | None = None,
    allow_input_schema_fallback: bool = True,
) -> dict[str, "LayerConfig"]:
    """Convert MoE weights from checkpoint format to Humming kernel format.

    This function processes weights for each sublayer (w13, w2) by:
    1. Converting from checkpoint format to humming format if needed
    2. Force requanting if a different quantization schema is specified
    3. Preparing layer metadata for the Humming kernel
    4. Transforming weights for inference

    Args:
        layer: The RoutedExperts layer containing weights to process
        quant_config: Optional quantization config dict. Required if weight_schema
                     or input_schema are None. Used to build schemas via
                     BaseWeightSchema.from_config().
        sublayer_configs: Optional configuration dict for each sublayer (w13, w2).
                         Each config must have "shape_n" and "shape_k" keys.
                         If None, configs are built from layer.moe_config properties.
        weight_schema: Optional initial weight quantization schema.
                      If None, built from quant_config.
        input_schema: Optional initial input quantization schema.
                     If None, built from quant_config or env vars.
        force_weight_schema: Optional schema to force requantization to
        allow_input_schema_fallback: Whether incompatible input schemas may be replaced.

    Side effects:
        - Modifies layer parameters in place
        - Sets layer.weight_schemas and layer.input_schemas
        - Sets layer.humming_configs for quant config construction

    """
    # Build schemas from quant_config if not provided
    has_bias = layer.moe_config.has_bias
    num_experts = layer.moe_config.num_local_experts
    param_dtype = layer.params_dtype

    if weight_schema is None or input_schema is None:
        if quant_config is None:
            raise ValueError(
                "Must provide either weight_schema/input_schema or quant_config"
            )

        from vllm.utils.humming import BaseWeightSchema, HummingInputSchema

        if weight_schema is None:
            weight_schema = BaseWeightSchema.from_config(quant_config)

        if input_schema is None:
            input_quant_config = (envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}).copy()
            if humming_is_layer_skipped(input_quant_config, layer.layer_name):
                input_schema = HummingInputSchema()
            else:
                # TODO: read input_quant_config from quant_config
                input_quant_config = humming_schema.resolve_humming_layer_config(
                    input_quant_config, layer.layer_name
                )
                allow_input_schema_fallback = input_quant_config.pop(
                    "allow_fallback", False
                )
                input_schema = HummingInputSchema.from_config(input_quant_config)

    # Build sublayer configs from layer properties if not provided
    if sublayer_configs is None:
        is_gated = layer.moe_config.activation.is_gated
        intermediate_size = layer.moe_config.intermediate_size_per_partition
        sublayer_configs = {
            "w13": {
                "shape_n": intermediate_size * (2 if is_gated else 1),
                "shape_k": layer.moe_config.hidden_dim,
            },
            "w2": {
                "shape_n": layer.moe_config.hidden_dim,
                "shape_k": intermediate_size,
            },
        }

    layer.weight_schemas = {}
    layer.input_schemas = {}
    humming_configs = {}

    for sublayer_name, configs in sublayer_configs.items():
        final_weight_schema, final_input_schema, humming_config = (
            _process_single_sublayer(
                layer=layer,
                sublayer_name=sublayer_name,
                shape_n=configs["shape_n"],
                shape_k=configs["shape_k"],
                weight_schema=weight_schema,
                input_schema=input_schema,
                has_bias=has_bias,
                num_experts=num_experts,
                param_dtype=param_dtype,
                force_weight_schema=force_weight_schema,
                allow_input_schema_fallback=allow_input_schema_fallback,
            )
        )

        layer.weight_schemas[sublayer_name] = final_weight_schema
        layer.input_schemas[sublayer_name] = final_input_schema
        humming_configs[sublayer_name] = humming_config

    layer.humming_configs = humming_configs
    return humming_configs

select_humming_moe_experts(config, weight_key, activation_key)

Select the primary Humming MoE Experts class Note: Shape-specific fallbacks may still occur at runtime.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def select_humming_moe_experts(
    config: FusedMoEConfig,
    weight_key: QuantKey | None,
    activation_key: QuantKey | None,
) -> type[mk.FusedMoEExperts] | None:
    """Select the primary Humming MoE Experts class
    Note: Shape-specific fallbacks may still occur at runtime.
    """
    if not has_humming():
        return None

    # NOTE: the kernels are selected in the following order.
    AVAILABLE_EXPERTS: list[type[mk.FusedMoEExperts]] = [
        BatchedHummingGroupedExperts,
        HummingGroupedExperts,
        HummingIndexedExperts,
    ]

    # NOTE(rob): We need to peak into the P/F selection to determine
    # if we are using the batched or standard expert format, which
    # if not ideal. Once we unify TP + DP/EP, we can select P/F first.
    activation_format = (
        mk.FusedMoEActivationFormat.BatchedExperts
        if config.moe_parallel_config.use_batched_activation_format
        else mk.FusedMoEActivationFormat.Standard
    )

    def _make_log_backend(experts_cls: type[mk.FusedMoEExperts]):
        return f"Using {experts_cls.__name__} Humming MoE backend."

    def _make_log_unsupported(
        experts_cls: type[mk.FusedMoEExperts], reason: str | None
    ) -> str:
        if reason:
            return (
                f"Humming MoE experts {experts_cls.__name__} does not support the "
                f"deployment configuration since {reason}."
            )
        else:
            return (
                f"Humming MoE experts '{experts_cls.__name__}' does not support the "
                "deployment configuration."
            )

    for k_cls in AVAILABLE_EXPERTS:
        supported, reason = k_cls.is_supported_config(
            k_cls,
            config,
            weight_key,
            activation_key,
            activation_format,
        )
        if supported:
            logger.info_once(_make_log_backend(k_cls))
            return k_cls
        else:
            logger.debug_once(_make_log_unsupported(k_cls, reason))

    return None

stage_moe_weights_for_humming(layer, w13, w2, w13_scale, w2_scale, w13_input_scale=None, w2_input_scale=None)

Install canonical weights and scales for Humming's layer-based converter.

Returns the scale parameter name used, weight_scale or weight_scale_inv.

Source code in vllm/model_executor/layers/quantization/utils/humming/moe.py

def stage_moe_weights_for_humming(
    layer: "RoutedExperts",
    w13: torch.Tensor,
    w2: torch.Tensor,
    w13_scale: torch.Tensor,
    w2_scale: torch.Tensor,
    w13_input_scale: torch.Tensor | None = None,
    w2_input_scale: torch.Tensor | None = None,
) -> str:
    """Install canonical weights and scales for Humming's layer-based converter.

    Returns the scale parameter name used, ``weight_scale`` or
    ``weight_scale_inv``.
    """
    scale_name = (
        "weight_scale_inv" if hasattr(layer, "w13_weight_scale_inv") else "weight_scale"
    )
    replace_parameter(layer, "w13_weight", w13)
    replace_parameter(layer, "w2_weight", w2)
    replace_parameter(layer, f"w13_{scale_name}", w13_scale)
    replace_parameter(layer, f"w2_{scale_name}", w2_scale)
    if w13_input_scale is not None:
        replace_parameter(layer, "w13_input_scale", w13_input_scale)
    if w2_input_scale is not None:
        replace_parameter(layer, "w2_input_scale", w2_input_scale)
    return scale_name