[bug] Encountered an error in forwardAsync function: Assertion failed: mNextBlocks.empty() #2708
Assignees
Labels
Comments
akhoroshev
changed the title
This was referenced Jan 21, 2025
akhoroshev
changed the title
|
Hi @Funatiq could u plz take a look this issue? |
github-actions
bot
added
triaged
|
@nv-guomingz @Funatiq Could you also look at these issues? |
|
this is also reproduced in the version d93a2dde84ead |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Errors happens under load after some time.
Executor API
version 548b5b7310
config.json
{ "version": "0.16.0.dev2024120300", "pretrained_config": { "architecture": "DeepseekForCausalLM", "dtype": "bfloat16", "vocab_size": 42064, "hidden_size": 2048, "num_hidden_layers": 28, "num_attention_heads": 16, "hidden_act": "swiglu", "logits_dtype": "float32", "norm_epsilon": 1e-05, "runtime_defaults": null, "position_embedding_type": "rope_gpt_neox", "num_key_value_heads": 8, "intermediate_size": 14336, "max_position_embeddings": 131072, "mapping": { "world_size": 1, "gpus_per_node": 8, "cp_size": 1, "tp_size": 1, "pp_size": 1, "moe_tp_size": 1, "moe_ep_size": 1 }, "quantization": { "quant_algo": null, "kv_cache_quant_algo": null, "group_size": 128, "smoothquant_val": 0.5, "clamp_val": null, "use_meta_recipe": false, "has_zero_point": false, "pre_quant_scale": false, "exclude_modules": null }, "use_parallel_embedding": false, "embedding_sharding_dim": 0, "share_embedding_table": false, "head_size": 128, "qk_layernorm": false, "rotary_embedding_dim": 128, "return_context_hidden": false, "logits_type": "float32", "moe_intermediate_size": 1792, "rotary_base": 300000, "rotary_scaling": null, "moe": { "num_experts": 64, "shared_expert_intermediate_size": 3584, "top_k": 6, "normalization_mode": 0 } }, "build_config": { "max_input_len": 130048, "max_seq_len": 131072, "opt_batch_size": 8, "max_batch_size": 256, "max_beam_width": 1, "max_num_tokens": 4096, "opt_num_tokens": 256, "max_prompt_embedding_table_size": 0, "kv_cache_type": "PAGED", "gather_context_logits": false, "gather_generation_logits": false, "strongly_typed": true, "force_num_profiles": null, "profiling_verbosity": "layer_names_only", "enable_debug_output": false, "max_draft_len": 0, "speculative_decoding_mode": 1, "use_refit": false, "input_timing_cache": null, "output_timing_cache": "model.cache", "lora_config": { "lora_dir": [], "lora_ckpt_source": "hf", "max_lora_rank": 64, "lora_target_modules": [], "trtllm_modules_to_hf_modules": {} }, "auto_parallel_config": { "world_size": 1, "gpus_per_node": 8, "cluster_key": "H100-PCIe", "cluster_info": null, "sharding_cost_model": "alpha_beta", "comm_cost_model": "alpha_beta", "enable_pipeline_parallelism": false, "enable_shard_unbalanced_shape": false, "enable_shard_dynamic_shape": false, "enable_reduce_scatter": true, "builder_flags": null, "debug_mode": false, "infer_shape": true, "validation_mode": false, "same_buffer_io": { "past_key_value_(\\d+)": "present_key_value_\\1" }, "same_spec_io": {}, "sharded_io_allowlist": [ "past_key_value_\\d+", "present_key_value_\\d*" ], "fill_weights": false, "parallel_config_cache": null, "profile_cache": null, "dump_path": null, "debug_outputs": [] }, "weight_sparsity": false, "weight_streaming": false, "plugin_config": { "dtype": "bfloat16", "bert_attention_plugin": "auto", "gpt_attention_plugin": "auto", "gemm_plugin": "bfloat16", "gemm_swiglu_plugin": null, "fp8_rowwise_gemm_plugin": null, "smooth_quant_gemm_plugin": null, "qserve_gemm_plugin": null, "identity_plugin": null, "layernorm_quantization_plugin": null, "rmsnorm_quantization_plugin": null, "nccl_plugin": null, "lora_plugin": null, "weight_only_groupwise_quant_matmul_plugin": null, "weight_only_quant_matmul_plugin": null, "smooth_quant_plugins": true, "quantize_per_token_plugin": false, "quantize_tensor_plugin": false, "moe_plugin": "auto", "mamba_conv1d_plugin": "auto", "low_latency_gemm_plugin": null, "low_latency_gemm_swiglu_plugin": null, "context_fmha": true, "bert_context_fmha_fp32_acc": false, "paged_kv_cache": true, "remove_input_padding": true, "reduce_fusion": false, "user_buffer": false, "tokens_per_block": 64, "use_paged_context_fmha": true, "use_fp8_context_fmha": false, "multiple_profiles": false, "paged_state": false, "streamingllm": false, "manage_weights": false, "use_fused_mlp": true, "pp_reduce_scatter": false }, "use_strip_plan": false, "max_encoder_input_len": 1024, "use_fused_mlp": true, "monitor_memory": false, "use_mrope": false } }Chunked context and context reuse are enabled.
Starting logs
The text was updated successfully, but these errors were encountered: