~/nemo-speech$ export VLLM_PLUGINS=nemo_speechlm
export VLLM_MAX_AUDIO_DECODE_DURATION_S=3600
export VLLM_MAX_AUDIO_CLIP_FILESIZE_MB=180
export VLLM_USE_V2_MODEL_RUNNER=0
vllm serve ./models/Nemotron-3.5-Transcribe-NVFP4 \
--served-model-name nemotron-transcribe \
--trust-remote-code \
--gpu-memory-utilization 0.90 \
--max-model-len 32768 --max-num-batched-tokens 4096 --max-num-seqs 16 \
--enable-chunked-prefill --no-enable-prefix-caching \
--use-replayssm --mamba-backend flashinfer --mamba-cache-mode none \
--mm-processor-cache-gb 0 --no-async-scheduling \
--limit-mm-per-prompt '{"audio":{"count":1,"length":120000000}}' \
--speculative-config '{"method":"mtp","num_speculative_tokens":2}' \
--reasoning-parser nemotron_v3 \
--trust-request-mm-kwargs \
--middleware nemo.collections.speechlm2.vllm.salm.ctc_serving.ctc_timestamp_middleware \
--host 0.0.0.0 --port 8000
OneLogger: Setting error_handling_strategy to DISABLE_QUIETLY_AND_REPORT_METRIC_ERROR for rank (rank=0) with OneLogger disabled. To override: explicitly set error_handling_strategy parameter.
No exporters were provided. This means that no telemetry data will be collected.
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1092: SyntaxWarning: invalid escape sequence '\.'
"""Normalize input text using a regular expression.
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1242: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[mM]r\.?", replacement="mister")
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1243: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[mM]s\.?", replacement="miss")
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1244: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[mM]rs\.?", replacement="missus")
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1245: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[mM]me\.?", replacement="madame")
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1246: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[dD]r\.?", replacement="doctor")
[NeMo W 2026-11-04 08:08:22 nemo_logging:365] /home/***/nemo-speech/nemo/collections/tts/parts/utils/tts_dataset_utils.py:1247: SyntaxWarning: invalid escape sequence '\.'
text = normalize_text_by_pattern(text=text, pattern="[eE]tc\.?", replacement="et cetera")
[NeMo W 2026-11-04 08:08:25 nemo_logging:365] /home/***/nemo-speech/.venv/lib/python3.12/site-packages/whisper_normalizer/basic.py:90: SyntaxWarning: invalid escape sequence '\X'
"""
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347]
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347] █ █ █▄ ▄█
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.32.0
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347] █▄█▀ █ █ █ █ model ./models/Nemotron-3.5-Transcribe-NVFP4
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:347]
(APIServer pid=555187) INFO 11-04 08:08:25 [api_utils.py:286] non-default args: {'model_tag': './models/Nemotron-3.5-Transcribe-NVFP4', 'trust_request_mm_kwargs': True, 'host': '0.0.0.0', 'middleware': ['nemo.collections.speechlm2.vllm.salm.ctc_serving.ctc_timestamp_middleware'], 'model': './models/Nemotron-3.5-Transcribe-NVFP4', 'trust_remote_code': True, 'max_model_len': 32768, 'served_model_name': ['nemotron-transcribe'], 'mamba_backend': 'flashinfer', 'reasoning_parser': 'nemotron_v3', 'gpu_memory_utilization': 0.9, 'enable_prefix_caching': False, 'use_replayssm': True, 'limit_mm_per_prompt': {'audio': {'count': 1, 'length': 120000000}}, 'mm_processor_cache_gb': 0.0, 'max_num_batched_tokens': 4096, 'max_num_seqs': 16, 'enable_chunked_prefill': True, 'async_scheduling': False, 'speculative_config': {'method': 'mtp', 'num_speculative_tokens': 2}}
(APIServer pid=555187) INFO 11-04 08:08:36 [model.py:733] Resolved architecture: Nemotron3_5TranscribeForConditionalGeneration
(APIServer pid=555187) INFO 11-04 08:08:36 [model.py:2152] Using max model len 32768
[transformers] The tokenizer you are loading from './models/Nemotron-3.5-Transcribe-NVFP4' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
(APIServer pid=555187) INFO 11-04 08:08:48 [model.py:733] Resolved architecture: Nemotron3_5TranscribeMTPModel
(APIServer pid=555187) INFO 11-04 08:08:48 [model.py:2152] Using max model len 262144
(APIServer pid=555187) WARNING 11-04 08:08:48 [speculative.py:1381] Enabling num_speculative_tokens > 1 will run multiple times of forward on same MTP layer,which may result in lower acceptance rate
(APIServer pid=555187) INFO 11-04 08:08:48 [speculative.py:1669] Overriding draft model max model len from 262144 to 32768
(APIServer pid=555187) INFO 11-04 08:08:48 [scheduler.py:307] Chunked prefill is enabled with max_num_batched_tokens=4096.
(APIServer pid=555187) INFO 11-04 08:08:48 [config.py:728] Updating mamba_ssm_cache_dtype to 'float32' for NemotronH model
(APIServer pid=555187) INFO 11-04 08:08:48 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native'])
(APIServer pid=555187) WARNING 11-04 08:08:48 [vllm.py:2276] max_num_scheduled_tokens is set to 4096 based on the speculative decoding settings. This may lead to suboptimal performance. Consider increasing max_num_batched_tokens to accommodate the additional draft token slots, or decrease num_speculative_tokens.
(APIServer pid=555187) INFO 11-04 08:08:48 [compilation.py:317] Enabled custom fusions: act_quant
(APIServer pid=555187) INFO 11-04 08:08:56 [base.py:280] Multi-modal warmup completed in 0.016s
OneLogger: Setting error_handling_strategy to DISABLE_QUIETLY_AND_REPORT_METRIC_ERROR for rank (rank=0) with OneLogger disabled. To override: explicitly set error_handling_strategy parameter.
No exporters were provided. This means that no telemetry data will be collected.
(EngineCore pid=555362) INFO 11-04 08:08:58 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='./models/Nemotron-3.5-Transcribe-NVFP4', speculative_config=SpeculativeConfig(method='mtp', model='./models/Nemotron-3.5-Transcribe-NVFP4', num_spec_tokens=2), tokenizer='./models/Nemotron-3.5-Transcribe-NVFP4', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=modelopt_mixed, quantization_config=None, enforce_eager=False, aux_output_config=AuxOutputConfig(enable_return_routed_experts=False, max_bytes=None), kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='nemotron_v3', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, custom_histogram_buckets=None, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=nemotron-transcribe, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::qwen4_exp_qsa_prepare_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'cudagraph_decoder_replay': True, 'decoder_replay_cudagraph_capture_sizes': [], 'decoder_replay_trim_threshold': 768, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 96, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_rocm_segmented_attn_autotune=False, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None)
(EngineCore pid=555362) INFO 11-04 08:08:59 [parallel_state.py:1906] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_6365fea8071a42dfb5372f1f7d3a2952 backend=nccl
(EngineCore pid=555362) INFO 11-04 08:08:59 [parallel_state.py:2350] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank 0, EPLB rank N/A
(EngineCore pid=555362) WARNING 11-04 08:08:59 [flashinfer.py:143] FlashInfer's precompiled kernels are missing or do not match flashinfer-python 0.7.0.post1 (flashinfer-cubin not installed, flashinfer-jit-cache not installed). FlashInfer then downloads and compiles kernels at startup, which can take several minutes, or fails to import mismatched ones. Run `vllm download-kernels` in this Python environment to install them.
(EngineCore pid=555362) INFO 11-04 08:09:02 [topk_topp_sampler.py:124] Using FlashInfer for top-p & top-k sampling.
(EngineCore pid=555362) WARNING 11-04 08:09:02 [__init__.py:211] min_p and logit_bias parameters won't work with speculative decoding.
[transformers] The tokenizer you are loading from './models/Nemotron-3.5-Transcribe-NVFP4' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
(EngineCore pid=555362) INFO 11-04 08:09:03 [gpu_model_runner.py:5039] Starting to load model ./models/Nemotron-3.5-Transcribe-NVFP4...
(EngineCore pid=555362) INFO 11-04 08:09:04 [__init__.py:797] Selected FlashInferFP8ScaledMMLinearKernel for MergedColumnParallelLinear
(EngineCore pid=555362) INFO 11-04 08:09:04 [deep_gemm.py:198] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
(EngineCore pid=555362) INFO 11-04 08:09:04 [deep_gemm.py:225] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
(EngineCore pid=555362) INFO 11-04 08:09:04 [deep_gemm.py:137] DeepGEMM E8M0 enabled on current platform.
(EngineCore pid=555362) INFO 11-04 08:09:04 [__init__.py:797] Selected FlashInferFP8ScaledMMLinearKernel for RowParallelLinear
(EngineCore pid=555362) INFO 11-04 08:09:04 [__init__.py:1240] Using FlashInferCutlassNvFp4LinearKernel for NVFP4 GEMM
(EngineCore pid=555362) INFO 11-04 08:09:04 [nvfp4.py:314] Using 'FLASHINFER_CUTLASS' NvFp4 MoE backend out of potential backends: ['FLASHINFER_TRTLLM', 'FLASHINFER_CUTEDSL', 'FLASHINFER_CUTEDSL_BATCHED', 'FLASHINFER_CUTLASS', 'VLLM_CUTLASS', 'MARLIN', 'HUMMING', 'EMULATION'].
(EngineCore pid=555362) INFO 11-04 08:09:04 [cuda.py:506] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
(EngineCore pid=555362) INFO 11-04 08:09:04 [flash_attn.py:1117] Using FlashAttention version 2
[NeMo W 2026-11-04 08:09:04 experimental:27] `<class 'nemo.collections.asr.modules.transformer_encoder.TransformerEncoder'>` is experimental and not ready for production yet. Use at your own risk.
[NeMo W 2026-11-04 08:09:04 experimental:27] `<class 'nemo.collections.asr.modules.parallel_expert_encoder.ParallelExpertEncoder'>` is experimental and not ready for production yet. Use at your own risk.
[NeMo W 2026-11-04 08:09:04 experimental:27] `<class 'nemo.collections.asr.modules.transformer_encoder.TransformerEncoder'>` is experimental and not ready for production yet. Use at your own risk.
[NeMo W 2026-11-04 08:09:04 sortformer_diar_models:484] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'validate_manifest_paths': False, 'sample_rate': 16000, 'num_spks': 8, 'session_len_sec': 105, 'soft_targets': False, 'batch_size': 42, 'shuffle': True, 'num_workers': 18, 'pin_memory': True, 'use_lhotse': False, 'subsegment_mode': True, 'subsegment_single_chunk_min_len_sec': 45, 'subsegment_two_chunks_rate': 0.5, 'subsegment_two_chunk_min_len_sec': 22, 'subsegment_nspk_bias': 1.5, 'subsegment_start_guard_sec': 0.25, 'subsegment_min_first_spk_sec': 0.5, 'subsegment_splice_silence_sec': 0.1, 'subsampling_factor': 8}
[NeMo W 2026-11-04 08:09:04 sortformer_diar_models:484] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'sample_rate': 16000, 'num_spks': 8, 'session_len_sec': -1, 'soft_targets': False, 'batch_size': 32, 'shuffle': False, 'num_workers': 18, 'drop_last': False, 'pin_memory': True, 'use_lhotse': False, 'subsampling_factor': 8}
[NeMo W 2026-11-04 08:09:04 sortformer_diar_models:484] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'sample_rate': 16000, 'num_spks': 8, 'session_len_sec': -1, 'soft_targets': False, 'batch_size': 32, 'shuffle': False, 'num_workers': 18, 'drop_last': False, 'pin_memory': True, 'use_lhotse': False, 'subsampling_factor': 8}
[NeMo W 2026-11-04 08:09:04 experimental:27] `<class 'nemo.collections.asr.modules.transformer_encoder.TransformerEncoder'>` is experimental and not ready for production yet. Use at your own risk.
[NeMo I 2026-11-04 08:09:05 fp8_encoder:175] FP8 audio encoder: 128 Linears built from quantization_config
[NeMo W 2026-11-04 08:09:05 experimental:27] `<class 'nemo.collections.asr.modules.transformer_encoder.TransformerEncoder'>` is experimental and not ready for production yet. Use at your own risk.
[NeMo I 2026-11-04 08:09:05 ctc_timestamps:1812] [NeMoSpeechLM] CTC timestamp inputs bound to vLLM encoder-cache identity.
[NeMo I 2026-11-04 08:09:05 model:261] [NeMoSpeechLM] CTC timestamps enabled from checkpoint config: /home/***/nemo-speech/models/Nemotron-3.5-Transcribe-NVFP4/model-ctc-bf16.safetensors
(EngineCore pid=555362) INFO 11-04 08:09:05 [base_loader.py:82] Initializing model took 1.855010 seconds
(EngineCore pid=555362) INFO 11-04 08:09:05 [weight_utils.py:1108] Filesystem type for checkpoints: EXT4. Checkpoint size: 20.08 GiB. Available RAM: 119.12 GiB.
(EngineCore pid=555362) INFO 11-04 08:09:05 [weight_utils.py:1131] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 20% Completed | 1/5 [00:00<00:02, 1.64it/s]
Loading safetensors checkpoint shards: 40% Completed | 2/5 [00:01<00:02, 1.20it/s]
Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:01<00:00, 3.84it/s]
Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:01<00:00, 2.94it/s]
(EngineCore pid=555362) INFO 11-04 08:09:13 [default_loader.py:501] Loading weights took 7.74 seconds
(EngineCore pid=555362) INFO 11-04 08:09:14 [nvfp4.py:629] Using MoEPrepareAndFinalizeNoDPEPModular
[rank0]:[W1104 08:09:14.301619077 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 331350016 bytes (free: 2818048, total: 33667612672).
[rank0]:[W1104 08:09:14.307879628 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 331350016 bytes (free: 313196544, total: 33667612672).
[rank0]:[W1104 08:09:14.313572094 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 331350016 bytes (free: 292225024, total: 33667612672).
[rank0]:[W1104 08:09:14.319648416 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 331350016 bytes (free: 271253504, total: 33667612672).
[rank0]:[W1104 08:09:14.325628726 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 331350016 bytes (free: 250281984, total: 33667612672).
(EngineCore pid=555362) INFO 11-04 08:09:14 [gpu_model_runner.py:5071] Loading drafter model...
(EngineCore pid=555362) INFO 11-04 08:09:14 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native'])
(EngineCore pid=555362) WARNING 11-04 08:09:14 [vllm.py:2276] max_num_scheduled_tokens is set to 4096 based on the speculative decoding settings. This may lead to suboptimal performance. Consider increasing max_num_batched_tokens to accommodate the additional draft token slots, or decrease num_speculative_tokens.
(EngineCore pid=555362) INFO 11-04 08:09:14 [compilation.py:317] Enabled custom fusions: act_quant
(EngineCore pid=555362) INFO 11-04 08:09:14 [__init__.py:797] Selected FlashInferFP8ScaledMMLinearKernel for QKVParallelLinear
(EngineCore pid=555362) INFO 11-04 08:09:14 [__init__.py:797] Selected FlashInferFP8ScaledMMLinearKernel for ColumnParallelLinear
(EngineCore pid=555362) INFO 11-04 08:09:14 [fp8.py:525] Using FLASHINFER_CUTLASS Fp8 MoE backend out of potential backends: ['AITER', 'FLASHINFER_TRTLLM', 'FLASHINFER_CUTLASS', 'DEEPGEMM', 'TRITON', 'MARLIN', 'HUMMING', 'BATCHED_DEEPGEMM', 'BATCHED_TRITON', 'XPU', 'CPU_W8A8', 'CPU', 'HPC'].
(EngineCore pid=555362) INFO 11-04 08:09:14 [base_loader.py:82] Initializing model took 0.092665 seconds
(EngineCore pid=555362) INFO 11-04 08:09:14 [weight_utils.py:825] Skipped 2/5 safetensors shard(s) holding no weights the model loads
(EngineCore pid=555362) INFO 11-04 08:09:14 [weight_utils.py:1108] Filesystem type for checkpoints: EXT4. Checkpoint size: 19.11 GiB. Available RAM: 119.17 GiB.
Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 33% Completed | 1/3 [00:00<00:00, 5.78it/s]
Loading safetensors checkpoint shards: 67% Completed | 2/3 [00:00<00:00, 5.83it/s]
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:00<00:00, 2.97it/s]
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:00<00:00, 3.42it/s]
(EngineCore pid=555362) INFO 11-04 08:09:16 [default_loader.py:501] Loading weights took 1.91 seconds
(EngineCore pid=555362) INFO 11-04 08:09:17 [flashinfer_utils.py:357] Padding intermediate size from 1856 to 1920 for up/down projection weights.
(EngineCore pid=555362) INFO 11-04 08:09:17 [fp8.py:856] Using MoEPrepareAndFinalizeNoDPEPModular
(EngineCore pid=555362) INFO 11-04 08:09:17 [llm_base_proposer.py:1469] Detected MTP model. Sharing target model embedding weights with the draft model.
(EngineCore pid=555362) INFO 11-04 08:09:17 [llm_base_proposer.py:1518] Detected EAGLE model without its own lm_head in the checkpoint. Sharing target model lm_head weights with the draft model.
(EngineCore pid=555362) INFO 11-04 08:09:18 [gpu_model_runner.py:5149] Model loading took 20.7 GiB memory and 13.362626 seconds
(EngineCore pid=555362) INFO 11-04 08:09:18 [torch_utils.py:295] Reducing Torch threads from 8 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
(EngineCore pid=555362) INFO 11-04 08:09:18 [interface.py:1096] Setting attention block size to 2304 tokens to ensure that attention page size is >= mamba page size.
(EngineCore pid=555362) INFO 11-04 08:09:18 [interface.py:1120] Padding mamba page size by 0.05% to ensure that mamba page size and attention page size are exactly equal.
(EngineCore pid=555362) INFO 11-04 08:09:18 [utils.py:347] Using LBNHC KV cache layout.
(EngineCore pid=555362) INFO 11-04 08:09:18 [gpu_model_runner.py:6165] Encoder cache will be initialized with a budget of 32704 tokens, and profiled with 1 audio items of the maximum feature size.
[NeMo W 2026-11-04 08:09:18 sortformer_modules:241] spkcache_update_period (300) is less than chunk_len (500). The effective update period will be 500.
PEE online inference: 100%|█████████████████████████████████████████████████████████████| 66/66 [00:02<00:00, 27.25it/s]
(EngineCore pid=555362) INFO 11-04 08:09:24 [monitor.py:40] [backbone] Starting torch.compile
(EngineCore pid=555362) INFO 11-04 08:09:29 [backends.py:1104] [backbone] Using cache directory: /home/***/.cache/vllm/torch_compile_cache/20ab445ecd/rank_0_0/backbone for vLLM's torch.compile
(EngineCore pid=555362) INFO 11-04 08:09:29 [backends.py:1166] [backbone] Dynamo bytecode transform time: 4.67 s
(EngineCore pid=555362) INFO 11-04 08:09:35 [backends.py:398] [backbone] Compiling a graph for compile range (1, 4096) takes 5.27 s
(EngineCore pid=555362) INFO 11-04 08:09:37 [backends.py:927] collected artifacts: 30 entries, 7 artifacts, 8004898 bytes total
(EngineCore pid=555362) INFO 11-04 08:09:37 [decorators.py:743] [backbone] saved AOT compiled function to /home/***/.cache/vllm/torch_compile_cache/torch_aot_compile/f564c2e4b02f3d3d4f8f5b6f8984bb05f320165d4066bcc2244634f7dc02eb65/rank_0_0/model
(EngineCore pid=555362) INFO 11-04 08:09:37 [monitor.py:62] [backbone] torch.compile took 12.38 s in total
Skipping cuDNN in bmm_fp8 auto candidates on SM12x: override_shape unavailable (cuDNN backend < 9.23.1); policy=ALL per-shape build is unbounded at serving time.
2026-11-04 08:09:37,363 - INFO - core.py:564 - flashinfer.jit: Building JIT module gemm_sm120; this can take several minutes on first use.
2026-11-04 08:12:49,978 - INFO - core.py:564 - flashinfer.jit: Building JIT module gemm; this can take several minutes on first use.
(EngineCore pid=555362) INFO 11-04 08:13:08 [mamba_mixer2.py:622] Warming up Mamba2 SSD Triton kernels...
2026-11-04 08:13:49,953 - INFO - core.py:564 - flashinfer.jit: Building JIT module fp4_gemm_cutlass_sm120; this can take several minutes on first use.
2026-11-04 08:15:35,970 - INFO - core.py:564 - flashinfer.jit: Building JIT module fused_moe_120; this can take several minutes on first use.