Inference Serving Configurations

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 16 GPUs across 4 nodes:
# 1x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-tp8-dcp8-1p1d-mooncake-hicache-r3-maxreq192-v1/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --hicache-io-backend direct
  --hicache-ratio 3
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.5
  --max-running-requests 192
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 1.89
  --max-running-requests 192
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 16 GPUs across 4 nodes:
# 1x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-tp8-ep8-dspark7-replayssm-1p1d-mooncake-hicache-r1-v1/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --expert-parallel-size 8
  --hicache-io-backend direct
  --hicache-ratio 1
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.14
  --max-running-requests 256
  --mem-fraction-static 0.85
  --moe-a2a-backend megamoe
  --moe-runner-backend deep_gemm
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --speculative-algorithm DSPARK
  --speculative-draft-model-path /home/_shared/aa-artifacts-dynamo-kimi-k3/models/Kimi-K3-DSpark/56ce616ad7486f0e96cbb51ef23ed5a1bce1d92d
  --speculative-dspark-block-size 7
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_DG_CACHE_DIR=/tmp/k3-deep-gemm-${USER}-${STAMP}
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=2048
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-linear-replayssm-spec
  --enable-symm-mem
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.5
  --max-running-requests 256
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --speculative-algorithm DSPARK
  --speculative-draft-model-path /home/_shared/aa-artifacts-dynamo-kimi-k3/models/Kimi-K3-DSpark/56ce616ad7486f0e96cbb51ef23ed5a1bce1d92d
  --speculative-dspark-block-size 7
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  DYNAMO_SGLANG_EXTRA_ARGS=--enable-linear-replayssm-spec
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_DG_CACHE_DIR=/tmp/k3-deep-gemm-${USER}-${STAMP}
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 16 GPUs across 4 nodes:
# 1x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-tp8-ep8-dspark7-replayssm-1p1d-mooncake-hicache-r8-v1/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --expert-parallel-size 8
  --hicache-io-backend direct
  --hicache-ratio 8
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.14
  --max-running-requests 256
  --mem-fraction-static 0.85
  --moe-a2a-backend megamoe
  --moe-runner-backend deep_gemm
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --speculative-algorithm DSPARK
  --speculative-draft-model-path /home/_shared/aa-artifacts-dynamo-kimi-k3/models/Kimi-K3-DSpark/56ce616ad7486f0e96cbb51ef23ed5a1bce1d92d
  --speculative-dspark-block-size 7
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_DG_CACHE_DIR=/tmp/k3-deep-gemm-${USER}-${STAMP}
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=2048
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-linear-replayssm-spec
  --enable-symm-mem
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.5
  --max-running-requests 256
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --speculative-algorithm DSPARK
  --speculative-draft-model-path /home/_shared/aa-artifacts-dynamo-kimi-k3/models/Kimi-K3-DSpark/56ce616ad7486f0e96cbb51ef23ed5a1bce1d92d
  --speculative-dspark-block-size 7
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  DYNAMO_SGLANG_EXTRA_ARGS=--enable-linear-replayssm-spec
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_DG_CACHE_DIR=/tmp/k3-deep-gemm-${USER}-${STAMP}
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf7) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 24 GPUs across 6 nodes:
# 2x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-2p1d-hicache-mooncake/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --hicache-io-backend direct
  --hicache-ratio 5
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --kv-events-config {'enable_kv_cache_events': True, 'endpoint': 'tcp://*:20081', 'publisher': 'zmq', 'topic': 'kv-events'}
  --mamba-full-memory-ratio 0.5
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 1.89
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf7) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 16 GPUs across 4 nodes:
# 1x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-1p1d-hicache-mooncake/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --hicache-io-backend direct
  --hicache-ratio 5
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 0.5
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 1.89
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf7) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 24 GPUs across 6 nodes:
# 2x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-2p1d-matched-pp2-mooncake/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --chunked-prefill-size 16384
  --context-length 262144
  --dcp-size 4
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --enable-hierarchical-cache
  --hicache-io-backend direct
  --hicache-ratio 5
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --kv-events-config {'enable_kv_cache_events': True, 'endpoint': 'tcp://*:20081', 'publisher': 'zmq', 'topic': 'kv-events'}
  --mamba-full-memory-ratio 0.5
  --max-prefill-tokens 16384
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 2
  --reasoning-parser kimi_k3
  --tensor-parallel-size 4
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --dcp-size 4
  --decode-log-interval 1
  --disaggregation-transfer-backend mooncake
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 1.89
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 2
  --reasoning-parser kimi_k3
  --tensor-parallel-size 4
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  MC_FORCE_MNNVL=1
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# NVIDIA Dynamo 1.5.0.dev20260816 (SGLang d59c1ddf7) disaggregated serving of
# Kimi-K3 (MXFP4) on GB300 NVL72 — 24 GPUs across 6 nodes:
# 2x prefill worker + 1x decode worker at 8 GPUs each, behind a Dynamo
# frontend (fastokens tokenizer, kimi_k3 tool-call and reasoning parsers).
#
# Launched by agentperf-client submissions/nvidia/kimi-k3/gb300-dynamo-sglang-2p1d-hicache-nixl/run.sh,
# which renders the worker YAML below and submits it through bench_sflow.
# Container: sglang-d59c1ddf-dynamo-1.5.0.dev20260816.sqsh

# Prefill worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-decode-extra-slots 16
  --disaggregation-transfer-backend nixl
  --enable-hierarchical-cache
  --hicache-io-backend direct
  --hicache-ratio 5
  --hicache-write-policy write_back
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --kv-events-config {'enable_kv_cache_events': True, 'endpoint': 'tcp://*:20081', 'publisher': 'zmq', 'topic': 'kv-events'}
  --mamba-full-memory-ratio 0.5
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Prefill worker environment:
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Decode worker:
  --context-length 262144
  --dcp-size 8
  --decode-log-interval 1
  --disaggregation-decode-extra-slots 16
  --disaggregation-transfer-backend nixl
  --host 0.0.0.0
  --kv-cache-dtype fp8_e4m3
  --mamba-full-memory-ratio 1.89
  --max-running-requests 512
  --mem-fraction-static 0.85
  --pipeline-parallel-size 1
  --reasoning-parser kimi_k3
  --tensor-parallel-size 8
  --tool-call-parser kimi_k3
  --trust-remote-code

# Decode worker environment:
  NCCL_CUMEM_ENABLE=1
  NCCL_MNNVL_ENABLE=1
  SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
  SGLANG_MAMBA_SSM_DTYPE=bfloat16
  UCX_NET_DEVICES=all
  UCX_TLS=rc,cuda_copy

# Kimi K3 (MXFP4) on B300 x8 — SGLang low-latency profile, TP=8, no speculative decoding.
docker run --gpus all --shm-size 32g --network host --ipc=host \
  lmsysorg/sglang:kimi-k3 sglang serve \
    --trust-remote-code --model-path moonshotai/Kimi-K3 --tp-size 8 \
    --disable-custom-all-reduce --mem-fraction-static 0.85 \
    --reasoning-parser kimi_k3 --tool-call-parser kimi_k3 --max-running-requests 256 \
    --enable-cache-report --enable-metrics --enable-symm-mem

# Kimi K3 (MXFP4) on B300 x8 — SGLang low-latency profile, TP=8, DSPARK speculative decoding.
docker run --gpus all --shm-size 32g --network host --ipc=host \
  lmsysorg/sglang:kimi-k3 sglang serve \
    --trust-remote-code --model-path moonshotai/Kimi-K3 --tp-size 8 \
    --disable-custom-all-reduce --mem-fraction-static 0.85 \
    --reasoning-parser kimi_k3 --tool-call-parser kimi_k3 --max-running-requests 256 \
    --enable-cache-report --enable-metrics --enable-symm-mem \
    --speculative-algorithm DSPARK --speculative-draft-model-path RadixArk/Kimi-K3-DSpark \
    --speculative-dspark-block-size 7 --enable-linear-replayssm-spec

# Kimi K3 (MXFP4) on B300 x8 — SGLang balanced profile, TP=8 / DCP=8, no speculative decoding.
docker run --gpus all --shm-size 32g --network host --ipc=host \
  lmsysorg/sglang:kimi-k3 sglang serve \
    --trust-remote-code --model-path moonshotai/Kimi-K3 --tp-size 8 \
    --disable-custom-all-reduce --mem-fraction-static 0.85 \
    --reasoning-parser kimi_k3 --tool-call-parser kimi_k3 --max-running-requests 256 \
    --enable-cache-report --enable-metrics --dcp-size 8

# Kimi K3 (MXFP4) on GB300 NVL72 — SGLang, TP=8 across 2 trays (8 GPUs), DSPARK speculative decoding. One launch per node.
sglang serve \
  --trust-remote-code \
  --max-running-requests 256 \
  --model-path /models/Kimi-K3 \
  --tp-size 8 \
  --nnodes 2 \
  --node-rank <0..1> \
  --dist-init-addr <head-ip>:20000 \
  --mem-fraction-static 0.85 \
  --context-length 262144 \
  --reasoning-parser kimi_k3 \
  --tool-call-parser kimi_k3 \
  --mamba-full-memory-ratio 0.22 \
  --host 0.0.0.0 \
  --port 30000 \
  --speculative-algorithm DSPARK \
  --speculative-draft-model-path /models/Kimi-K3-DSpark \
  --speculative-dspark-block-size 7 \
  --enable-linear-replayssm-spec

# Kimi K3 (MXFP4) on GB300 NVL72 — SGLang, TP=8 / DCP=8 across 2 trays (8 GPUs), no speculative decoding. One launch per node.
sglang serve \
  --trust-remote-code \
  --max-running-requests 256 \
  --model-path /models/Kimi-K3 \
  --tp-size 8 \
  --nnodes 2 \
  --node-rank <0..1> \
  --dist-init-addr <head-ip>:20000 \
  --dcp-size 8 \
  --mem-fraction-static 0.85 \
  --context-length 262144 \
  --reasoning-parser kimi_k3 \
  --tool-call-parser kimi_k3 \
  --mamba-full-memory-ratio 1.89 \
  --host 0.0.0.0 \
  --port 30000

# Kimi K3 (MXFP4) on GB300 NVL72 — SGLang, TP=8 across 2 trays (8 GPUs), no speculative decoding. One launch per node.
sglang serve \
  --trust-remote-code \
  --max-running-requests 256 \
  --model-path /models/Kimi-K3 \
  --tp-size 8 \
  --nnodes 2 \
  --node-rank <0..1> \
  --dist-init-addr <head-ip>:20000 \
  --mem-fraction-static 0.85 \
  --context-length 262144 \
  --reasoning-parser kimi_k3 \
  --tool-call-parser kimi_k3 \
  --mamba-full-memory-ratio 0.24 \
  --host 0.0.0.0 \
  --port 30000

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) — B300 x8: TP8 + DP8 attention, EAGLE(k1,d3,s2 decode), maxreq 256, DP-routed workload.
# Single node, 8 GPUs. NVIDIA baseline flags verbatim.
# WORKLOAD REQUIREMENT: the benchmark client pins each conversation to one
# attention-DP rank (SGLang routed_dp_rank; agentperf sgl_conversation_dp_size: 8).
# Without that affinity the multi-turn workload thrashes the prefix cache across
# DP ranks and decode is prefill-starved to a fraction of these numbers.

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
cuda-graph-max-bs: 16
data-parallel-size: 8
enable-dp-attention: true
max-running-requests: 256
speculative-algorithm: EAGLE
speculative-attention-mode: decode
speculative-eagle-topk: 1
speculative-num-draft-tokens: 3
speculative-num-steps: 2

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) — B300 x8: TP8 + DP8 attention, no spec decode, maxreq 1024, DP-routed workload.
# Single node, 8 GPUs. NVIDIA baseline flags verbatim.
# WORKLOAD REQUIREMENT: the benchmark client pins each conversation to one
# attention-DP rank (SGLang routed_dp_rank; agentperf sgl_conversation_dp_size: 8).
# Without that affinity the multi-turn workload thrashes the prefix cache across
# DP ranks and decode is prefill-starved to a fraction of these numbers.

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
data-parallel-size: 8
enable-dp-attention: true
max-running-requests: 1024

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) — GB300 2x4: TP8 across trays, EAGLE(k1,d6,s5), maxreq 16 — low-latency.
# 2 nodes x 4 GPUs (--nnodes 2). NVIDIA baseline flags verbatim.

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
cuda-graph-max-bs: 16
max-prefill-tokens: 8192
max-running-requests: 16
speculative-algorithm: EAGLE
speculative-eagle-topk: 1
speculative-num-draft-tokens: 6
speculative-num-steps: 5

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) — GB300 2x4: TP8 across trays + DP8 attention, no spec decode, maxreq 1024, DP-routed workload.
# 2 nodes x 4 GPUs (--nnodes 2). NVIDIA baseline flags verbatim.
# WORKLOAD REQUIREMENT: the benchmark client pins each conversation to one
# attention-DP rank (SGLang routed_dp_rank; agentperf sgl_conversation_dp_size: 8).
# Without that affinity the multi-turn workload thrashes the prefix cache across
# DP ranks and decode is prefill-starved to a fraction of these numbers.

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
data-parallel-size: 8
enable-dp-attention: true
max-running-requests: 1024

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) on B300 x8 — TP8, EAGLE speculative decode (topk 1, 6 draft tokens, 5 steps), max-running-requests 16 — low-latency.
# Single node, 8 GPUs, aggregated in-flight batching. NVIDIA baseline config
# glm-5-2-sglang-ifb-tp8-eagle-low-latency-b300-v1 (bench_sflow).

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
cuda-graph-max-bs: 16
max-prefill-tokens: 8192
max-running-requests: 16
speculative-algorithm: EAGLE
speculative-eagle-topk: 1
speculative-num-draft-tokens: 6
speculative-num-steps: 5

# SGLang 0.5.17 (lmsysorg/sglang:v0.5.17-cu130, digest 16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155)
# serving GLM-5.2 (NVFP4) on B300 x8 — TP8, EAGLE speculative decode (topk 1, 6 draft tokens, 5 steps), max-running-requests 32.
# Single node, 8 GPUs, aggregated in-flight batching. NVIDIA baseline config
# glm-5-2-nvfp4-sglang-ifb-tp8-eagle5-maxreq32-bia-v1 (bench_sflow).

python3 -m sglang.launch_server \
  --config ifb_config.yaml \
  --model-path nvidia/GLM-5.2-NVFP4 \
  --served-model-name GLM-5.2 \
  --host 0.0.0.0 --port 20800 \
  --enable-cache-report

# ---------------- ifb_config.yaml (verbatim, verified against the running server) ----------------
chunked-prefill-size: 8192
mem-fraction-static: 0.85
quantization: modelopt_fp4
reasoning-parser: glm45
tensor-parallel-size: 8
tool-call-parser: glm47
cuda-graph-max-bs: 16
max-prefill-tokens: 8192
max-running-requests: 32
speculative-algorithm: EAGLE
speculative-eagle-topk: 1
speculative-num-draft-tokens: 6
speculative-num-steps: 5

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 12 GPUs across 3 nodes:
# 1x context worker (TP4 + MoE EP4, 4 GPUs) + 1x generation worker
# (TP8 + MoE EP8, MTP depth 5) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1 DYN_TRTLLM_ENABLE_ATTENTION_DP=1
#   DYN_TRTLLM_BATCHED_EGRESS=1 DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1
#   DYN_SYSTEM_PORT=8081 DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens
#   DYN_KV_BLOCK_SIZE=128 TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1
#   TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto01_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p01 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 8
moe_expert_parallel_size: 8
enable_attention_dp: false
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 4
max_num_tokens: 32
max_seq_len: 131072
print_iter_log: true
enable_autotuner: true
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.8
  dtype: fp8
  avg_seq_len: 39040
moe_config:
  backend: TRTLLM
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 8
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
allreduce_strategy: MNNVL
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
  use_cute_dsl_topk: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 48 GPUs across 12 nodes:
# 4x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP32 + MoE EP32, MTP depth 5) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto04_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p04 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 32
moe_expert_parallel_size: 32
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 4
max_num_tokens: 24
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 4
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 56 GPUs across 14 nodes:
# 6x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP32 + MoE EP32, MTP depth 5) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto05_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p05 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 32
moe_expert_parallel_size: 32
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 8
max_num_tokens: 48
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 5

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 6
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 48 GPUs across 12 nodes:
# 8x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP16 + MoE EP16, MTP depth 3) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 TRTLLM_ENABLE_KVCACHE_RECEIVE_PARALLEL=1
#   TRTLLM_KVCACHE_RECV_BUFFER_COUNT=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto07_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p07 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 16
moe_expert_parallel_size: 16
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 32
max_num_tokens: 192
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 8
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 64 GPUs across 16 nodes:
# 12x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP16 + MoE EP16, MTP depth 2) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 TRTLLM_ENABLE_KVCACHE_RECEIVE_PARALLEL=1
#   TRTLLM_KVCACHE_RECV_BUFFER_COUNT=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto08_img4_batched_egress1.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p08 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 16
moe_expert_parallel_size: 16
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 48
max_num_tokens: 288
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 48
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 2
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 2

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 12
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 68 GPUs across 17 nodes:
# 13x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP16 + MoE EP16, MTP depth 3) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 TRTLLM_ENABLE_KVCACHE_RECEIVE_PARALLEL=1
#   TRTLLM_KVCACHE_RECV_BUFFER_COUNT=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto09_img4_batched_egress1.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p09 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 16
moe_expert_parallel_size: 16
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 64
max_num_tokens: 384
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 48
  - 64
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 13
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 12 GPUs across 3 nodes:
# 1x context worker (TP4 + MoE EP4, 4 GPUs) + 1x generation worker
# (TP8 + MoE EP8, MTP depth 3) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto02_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p02 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 8
moe_expert_parallel_size: 8
enable_attention_dp: false
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 8
max_num_tokens: 48
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
allreduce_strategy: MNNVL
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# NVIDIA Dynamo (81eee4bb) + TensorRT-LLM (92d37d84) disaggregated serving of
# GLM-5.2 (NVFP4) on GB300 NVL72 — 40 GPUs across 10 nodes:
# 2x context workers (TP4 + MoE EP4, 4 GPUs each) + 1x generation worker
# (TP32 + MoE EP32, MTP depth 3) + Dynamo frontend with KV-cache-aware routing
# (fastokens tokenizer, glm47 tool-call parser, glm45 reasoning parser).

# Worker environment (every worker; container paths and per-allocation endpoints omitted):
#   DYN_TRTLLM_ENABLE_ATTENTION_DP=1 DYN_TRTLLM_BATCHED_EGRESS=1
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_SYSTEM_PORT=8081
#   DYN_UCX_TLS=^ib DYN_TCP_REQUEST_TIMEOUT=30 DYN_TOKENIZER=fastokens DYN_KV_BLOCK_SIZE=128
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TRTLLM_REQUEST_KV_CACHE_CONCURRENT=1
#   TLLM_PREFIX_TOKEN_CACHE=1 TRTLLM_SERVE_ENABLE_MSGSPEC=1
#   TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM=2 PYTHONUNBUFFERED=1 DYN_PUBLISH_KV_EVENTS=0
#   DYN_TOOL_CALL_PARSER=glm47 DYN_REASONING_PARSER=glm45 SERVED_MODEL_NAME=GLM-5.2

# Dynamo frontend / router environment (OpenAI-compatible on :8333):
#   ROUTER_MODE=kv DYN_KV_BLOCK_SIZE=128 DEBUG_ROUTER=0 PYTHONUNBUFFERED=1
#   DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_TEMPERATURE=0
#   DYN_ROUTER_TRACK_PREFILL_TOKENS=1 DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30
#   DYN_LOG=warn DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=2400

# Bring-up is driven by NVIDIA's submitter, which renders and launches the worker
# commands with the configs below:
python3 scripts/submit_agent_perf.py \
  --cluster-file clusters/aa-raplab.yaml \
  --config agent-perf/glm_5.2/gb300/config_raplab.yaml \
  --slo-yaml pareto03_img4_batched_egress1_fullpareto.yaml \
  --image-path-file image_path_img3.txt \
  --generated-dir generated_p03 \
  --results-dir results/dynamo/img3 \
  --wait-poll-seconds 30

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 32
moe_expert_parallel_size: 32
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
pipeline_parallel_size: 1
max_batch_size: 2
max_num_tokens: 12
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  avg_seq_len: 36020
moe_config:
  backend: CUTEDSL
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
stream_interval: 100
num_postprocess_workers: 4
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
sparse_attention_config:
  algorithm: dsa
  use_cute_dsl_paged_mqa_logits: true
  index_share_for_mtp_iteration: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.7
  dtype: fp8
  host_cache_size: 100000000000
  event_buffer_max_size: 0
  avg_seq_len: 39040
cache_transceiver_config:
  max_tokens_in_buffer: 524288
  backend: UCX
  transceiver_runtime: CPP
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 2
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware
    tokens_per_block: 128
    backfill_block_hashes_on_finish: true
gen_strip_message_history: true
gen_tokids_ctxbytes: true

# TensorRT-LLM (feat/deepseek_v4 @ ace71873) + NVIDIA Dynamo (@ 5a43d4ad) disaggregated
# serving on GB300 NVL72 — 32 GPUs (8 compute trays x 4):
# 6x context workers (TP4, attention-DP -> DP4 + MoE EP4, 4 GPUs each) + 1x generation
# worker (8 GPUs over 2 trays, attention-DP8 + MoE EP8) + Dynamo frontend with KV-aware
# conversation-affinity routing.

# Worker environment (every worker; CUDA_VISIBLE_DEVICES=0,1,2,3 per tray):
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVE_ENABLE_ORJSON=1 TRTLLM_SERVER_DISABLE_GC=1
#   TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1
#   DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=^ib
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_KV_BLOCK_SIZE=128
#   DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4
#   (context workers additionally: TRTLLM_SKIP_KV_CACHE_ESTIMATION=1)

# Generation worker (8 GPUs, MPI across 2 trays):
trtllm-llmapi-launch python3 -m dynamo.trtllm \
  --model-path <model-path> --served-model-name <model-path> \
  --extra-engine-args gen_config.yaml --disaggregation-mode decode \
  --request-plane tcp --event-plane zmq

# Context workers (x6, one per tray):
trtllm-llmapi-launch python3 -m dynamo.trtllm \
  --model-path <model-path> --served-model-name <model-path> \
  --extra-engine-args ctx_config.yaml --disaggregation-mode prefill \
  --request-plane tcp --event-plane zmq

# Dynamo frontend (router env: DYN_ROUTER_TEMPERATURE=0 DYN_ROUTER_QUEUE_THRESHOLD=None
#   DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=300 DYN_TOKENIZER=fastokens DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TCP_REQUEST_TIMEOUT=30):
python3 -m dynamo.frontend --http-port 8333 --router-mode kv --no-router-kv-events \
  --kv-cache-block-size 128 --enforce-disagg --router-reset-states \
  --request-plane tcp --event-plane zmq --dyn-tool-call-parser deepseek_v4

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 8
moe_expert_parallel_size: 8
enable_attention_dp: true
enable_lm_head_tp_in_adp: true
pipeline_parallel_size: 1
max_batch_size: 288
max_num_tokens: 1152
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.8
  dtype: fp8
  host_cache_size: 42949672960
  avg_seq_len: 39040
moe_config:
  backend: MEGAMOE_DEEPGEMM
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
stream_interval: 100
num_postprocess_workers: 8
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 64
  - 128
  - 192
  - 256
  - 272
  - 288
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
  allow_advanced_sampling: true
sparse_attention_config:
  algorithm: deepseek_v4
  enable_heuristic_topk: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.6
  dtype: fp8
  host_cache_size: 171798691840
  event_buffer_max_size: 0
  pool_ratio:
  - 0.3
  - 0.45
  - 0.25
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
context_servers:
  num_instances: 6
  router:
    type: kv_cache_aware
    tokens_per_block: 128
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware

# TensorRT-LLM (feat/deepseek_v4 @ ace71873) + NVIDIA Dynamo (@ 5a43d4ad) disaggregated
# serving on GB300 NVL72 — 36 GPUs (9 compute trays x 4):
# 5x context workers (TP4, attention-DP -> DP4 + MoE EP4, 4 GPUs each) + 1x generation
# worker (16 GPUs over 4 trays, attention-DP16 + MoE EP16) + Dynamo frontend with KV-aware
# conversation-affinity routing.

# Worker environment (every worker; CUDA_VISIBLE_DEVICES=0,1,2,3 per tray):
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVE_ENABLE_ORJSON=1 TRTLLM_SERVER_DISABLE_GC=1
#   TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0
#   TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 HF_HUB_OFFLINE=1
#   TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1
#   DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=^ib
#   DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_KV_BLOCK_SIZE=128
#   DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4
#   (context workers additionally: TRTLLM_SKIP_KV_CACHE_ESTIMATION=1)

# Generation worker (16 GPUs, MPI across 4 trays):
trtllm-llmapi-launch python3 -m dynamo.trtllm \
  --model-path <model-path> --served-model-name <model-path> \
  --extra-engine-args gen_config.yaml --disaggregation-mode decode \
  --request-plane tcp --event-plane zmq

# Context workers (x5, one per tray):
trtllm-llmapi-launch python3 -m dynamo.trtllm \
  --model-path <model-path> --served-model-name <model-path> \
  --extra-engine-args ctx_config.yaml --disaggregation-mode prefill \
  --request-plane tcp --event-plane zmq

# Dynamo frontend (router env: DYN_ROUTER_TEMPERATURE=0 DYN_ROUTER_QUEUE_THRESHOLD=None
#   DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=300 DYN_TOKENIZER=fastokens DYN_TOKENIZER_CACHE=1
#   DYN_TOKENIZER_CACHE_BYTES=8000000000 DYN_TCP_REQUEST_TIMEOUT=30
#   LD_PRELOAD=libjemalloc.so.2):
python3 -m dynamo.frontend --http-port 8333 --router-mode kv --no-router-kv-events \
  --kv-cache-block-size 128 --enforce-disagg --router-reset-states \
  --request-plane tcp --event-plane zmq --dyn-tool-call-parser deepseek_v4

# ---------------- gen_config.yaml ----------------
tensor_parallel_size: 16
moe_expert_parallel_size: 16
enable_attention_dp: true
enable_lm_head_tp_in_adp: true
pipeline_parallel_size: 1
max_batch_size: 44
max_num_tokens: 1024
max_seq_len: 131072
print_iter_log: true
enable_autotuner: false
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.8
  dtype: fp8
  host_cache_size: 42949672960
  avg_seq_len: 39040
moe_config:
  backend: MEGAMOE_DEEPGEMM
  use_low_precision_moe_combine: true
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
stream_interval: 100
num_postprocess_workers: 8
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 36
  - 40
  - 44
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
  allow_advanced_sampling: true
sparse_attention_config:
  algorithm: deepseek_v4
  enable_heuristic_topk: true
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
tensor_parallel_size: 4
moe_expert_parallel_size: 4
enable_attention_dp: true
pipeline_parallel_size: 1
enable_chunked_prefill: true
print_iter_log: true
cuda_graph_config: null
enable_autotuner: true
disable_overlap_scheduler: true
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.6
  dtype: fp8
  host_cache_size: 171798691840
  event_buffer_max_size: 0
  pool_ratio:
  - 0.3
  - 0.45
  - 0.25
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: false
  kv_cache_routing_conversation_affinity: true
  kv_cache_routing_max_sessions: 65536
return_perf_metrics: false
perf_metrics_max_requests: 200000
enable_iter_perf_stats: false
enable_iter_req_stats: false
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
context_servers:
  num_instances: 5
  router:
    type: kv_cache_aware
    tokens_per_block: 128
generation_servers:
  num_instances: 1
  router:
    type: kv_cache_aware

# TensorRT-LLM 1.3.0rc15.post1 disaggregated serving on a single 8x B300 node:
# 1x context worker (TP4 + EP4, attention-DP, GPUs 4-7) + 1x generation worker (TP4 + EP4,
# attention-DP, GPUs 0-3), colocated, + disagg orchestrator with conversation-affinity routing.
# SLURM allocation: --exclusive --cpus-per-task=28 (224 CPUs).

# Worker environment:
#   TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
#   TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10
#   TLLM_ADP_ROUTER_DECISION_LOG=1 TLLM_ADP_ROUTER_INCLUDE_TRANSFER_LOAD=1
#   TLLM_ADP_ROUTER_RANDOMIZE_TIEBREAK=1 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1
#   TRTLLM_MHC_ENABLE_FUSED_HC=0 UCX_TLS=rc,sm,self,cuda_copy,cuda_ipc
#   HF_HUB_OFFLINE=1 TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC=480

# Generation worker (CUDA_VISIBLE_DEVICES=0,1,2,3):
trtllm-serve <model-path> --config gen_config.yaml \
  --custom_tokenizer deepseek_v4 --tool_parser deepseek_v4

# Context worker (CUDA_VISIBLE_DEVICES=4,5,6,7):
trtllm-serve <model-path> --config ctx_config.yaml \
  --custom_tokenizer deepseek_v4 --tool_parser deepseek_v4

# Orchestrator:
trtllm-serve disaggregated -c server_config.yaml -t 7200 -r 7200

# ---------------- gen_config.yaml ----------------
backend: pytorch
tensor_parallel_size: 4
moe_expert_parallel_size: 4
pipeline_parallel_size: 1
enable_attention_dp: true
enable_lm_head_tp_in_adp: true
max_batch_size: 64
max_num_tokens: 1024
max_seq_len: 131072
enable_autotuner: false
print_iter_log: true
stream_interval: 100
num_postprocess_workers: 4
moe_config:
  backend: TRTLLM
  use_low_precision_moe_combine: true
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.8
  dtype: fp8
  host_cache_size: 0
  iteration_stats_interval: 1
  use_kv_cache_manager_v2: false
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 64
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
return_perf_metrics: false
perf_metrics_max_requests: 100000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- ctx_config.yaml ----------------
backend: pytorch
tensor_parallel_size: 4
moe_expert_parallel_size: 4
pipeline_parallel_size: 1
enable_attention_dp: true
max_batch_size: 128
max_num_tokens: 8192
max_seq_len: 131072
enable_chunked_prefill: true
disable_overlap_scheduler: true
enable_autotuner: true
print_iter_log: true
cuda_graph_config: null
moe_config:
  backend: TRTLLM
kv_cache_config:
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.8
  dtype: fp8
  host_cache_size: 214748364800
  event_buffer_max_size: 16384
  iteration_stats_interval: 1
  use_kv_cache_manager_v2: false
cache_transceiver_config:
  max_tokens_in_buffer: 131072
  backend: NIXL
  transceiver_runtime: PYTHON
scheduler_config:
  capacity_scheduler_policy: MAX_UTILIZATION
attention_dp_config:
  enable_kv_cache_aware_routing: true
  kv_cache_routing_load_balance_weight: 0.5
  kv_cache_routing_cold_start_warmup: true
  kv_cache_routing_match_rate_threshold: 0.1
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 3
return_perf_metrics: false
perf_metrics_max_requests: 100000
enable_iter_perf_stats: false
enable_iter_req_stats: false

# ---------------- server_config.yaml (topology; hosts/ports omitted) ----------------
backend: pytorch
context_servers:
  num_instances: 1
  router:
    type: conversation
generation_servers:
  num_instances: 1
  router:
    type: conversation

docker run --gpus all --shm-size 32g --network host \
  -v path/to/huggingface:/root/.cache/huggingface \
  lmsysorg/sglang:deepseek-v4-hopper \
  python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V4-Pro \
    --tp 8 --context-length 393216 --mem-fraction-static 0.88 \
    --speculative-algorithm EAGLE --speculative-num-steps 1 \
    --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \
    --moe-runner-backend marlin --host 0.0.0.0 --port 30000

docker run -d \
  --device=/dev/kfd --device=/dev/dri \
  --shm-size 32g --ipc=host --network=host --privileged \
  -v <host-path>:/model \
  -e SGLANG_DSV4_FP4_EXPERTS=True \
  -e SGLANG_FORCE_TRITON_MOE_FP8=0 \
  -e SGLANG_USE_AITER=1 \
  -e SGLANG_USE_ROCM700A=1 \
  -e SGLANG_HACK_FLASHMLA_BACKEND=triton \
  -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 \
  -e SGLANG_TOPK_TRANSFORM_512_TORCH=0 \
  -e SGLANG_OPT_USE_FUSED_COMPRESS=true \
  -e SGLANG_OPT_USE_OLD_COMPRESSOR=false \
  -e SGLANG_OPT_USE_AITER_MHC_PRE=true \
  -e SGLANG_OPT_USE_AITER_MHC_POST=true \
  -e SGLANG_OPT_USE_FUSED_STORE_CACHE=true \
  -e SGLANG_OPT_USE_OVERLAP_STORE_CACHE=false \
  -e SGLANG_OPT_USE_FUSED_PAGED_COMPRESS=true \
  -e SGLANG_OPT_FUSE_WQA_WKV=true \
  -e SGLANG_OPT_USE_TRITON_SWA_PREPARE=true \
  -e SGLANG_OPT_USE_FUSED_HASH_TOPK=true \
  -e SGLANG_OPT_DPSK_V4_RADIX=1 \
  -e SGLANG_OPT_USE_TILELANG_INDEXER=true \
  -e AITER_BF16_FP8_MOE_BOUND=1 \
  -e SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=0 \
  rocm/sgl-dev:rocm720-mi35x-f96ac98-20260527-DSv4 \
  python3 -m sglang.launch_server \
    --model-path /model \
    --served-model-name deepseek-ai/DeepSeek-V4-Pro \
    --tp 8 \
    --port 30000 \
    --host 0.0.0.0 \
    --trust-remote-code \
    --attention-backend compressed \
    --max-running-requests 256 \
    --page-size 256 \
    --context-length 32768 \
    --mem-fraction-static 0.90 \
    --swa-full-tokens-ratio 0.15 \
    --chunked-prefill-size 8192 \
    --disable-shared-experts-fusion \
    --tool-call-parser deepseekv4 \
    --reasoning-parser deepseek-v4

docker run -d \
  --device=/dev/kfd --device=/dev/dri \
  --shm-size 32g --ipc=host --network=host --privileged \
  -v <host-path>:/model \
  -e SGLANG_DSV4_FP4_EXPERTS=True \
  -e SGLANG_FORCE_TRITON_MOE_FP8=0 \
  -e SGLANG_USE_AITER=1 \
  -e SGLANG_USE_ROCM700A=1 \
  -e SGLANG_HACK_FLASHMLA_BACKEND=triton \
  -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 \
  -e SGLANG_TOPK_TRANSFORM_512_TORCH=0 \
  -e SGLANG_OPT_USE_FUSED_COMPRESS=true \
  -e SGLANG_OPT_USE_OLD_COMPRESSOR=false \
  -e SGLANG_OPT_USE_AITER_MHC_PRE=true \
  -e SGLANG_OPT_USE_AITER_MHC_POST=true \
  -e SGLANG_OPT_USE_FUSED_STORE_CACHE=true \
  -e SGLANG_OPT_USE_OVERLAP_STORE_CACHE=false \
  -e SGLANG_OPT_USE_FUSED_PAGED_COMPRESS=true \
  -e SGLANG_OPT_FUSE_WQA_WKV=true \
  -e SGLANG_OPT_USE_TRITON_SWA_PREPARE=true \
  -e SGLANG_OPT_USE_FUSED_HASH_TOPK=true \
  -e SGLANG_OPT_DPSK_V4_RADIX=1 \
  -e SGLANG_OPT_USE_TILELANG_INDEXER=true \
  -e AITER_BF16_FP8_MOE_BOUND=1 \
  -e SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=0 \
  rocm/sgl-dev:rocm720-mi35x-3b59630-20260526-DSv4 \
  python3 -m sglang.launch_server \
    --model-path /model \
    --served-model-name deepseek-ai/DeepSeek-V4-Pro \
    --tp 8 \
    --port 30000 \
    --host 0.0.0.0 \
    --trust-remote-code \
    --attention-backend compressed \
    --max-running-requests 256 \
    --page-size 256 \
    --chunked-prefill-size 8192 \
    --disable-shared-experts-fusion \
    --tool-call-parser deepseekv4 \
    --reasoning-parser deepseek-v4

#!/bin/bash
# gpt-oss-120b (FP4) on B200 — TensorRT-LLM 1.3.0rc12.post1 + NVIDIA Dynamo, disaggregated:
# 1x prefill worker (GPU0, TP1) + 3x independent decode workers (GPU1..3, TP1 each),
# Dynamo frontend (:8000) with KV-aware routing across decode replicas.
# EAGLE3 speculative decoding (3 draft tokens) on both roles.
set -u
export DYN_REQUEST_PLANE=tcp
export DYN_EVENT_PLANE=zmq
export UCX_TLS=cuda_ipc,cuda_copy,tcp,self
export DYN_REASONING_PARSER=gpt_oss
export DYN_TOOL_CALL_PARSER=harmony
export ETCD_ENDPOINTS=http://localhost:2379
MODEL_PATH=openai/gpt-oss-120b
SERVED=gpt-oss-120b

etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://localhost:2379 &
sleep 4

# Prefill worker (GPU0, TP1)
CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.trtllm \
  --model-path "$MODEL_PATH" --served-model-name "$SERVED" \
  --disaggregation-mode prefill --extra-engine-args /work/nsc_prefill_engine.yaml \
  --request-plane tcp --event-plane zmq &

# 3x decode workers (GPU1..3, TP1 each)
for g in 1 2 3; do
  CUDA_VISIBLE_DEVICES=$g python3 -m dynamo.trtllm \
    --model-path "$MODEL_PATH" --served-model-name "$SERVED" \
    --disaggregation-mode decode --extra-engine-args /work/nsc_decode_engine.yaml \
    --request-plane tcp --event-plane zmq &
done

# Dynamo frontend (KV router, :8000)
python3 -m dynamo.frontend --http-port 8000 --request-plane tcp --event-plane zmq \
  --router-mode kv --no-router-kv-events --router-ttl-secs 2400.0 &

#!/bin/bash
# gpt-oss-120b (FP4) on B200 — TensorRT-LLM 1.3.0rc12.post1 + NVIDIA Dynamo, disaggregated:
# 1x prefill worker (GPU0, TP1) + 1x decode worker (GPU1..4, TP4 via
# mpirun trtllm-llmapi-launch per the NVIDIA recipe), Dynamo frontend (:8000)
# with KV-aware routing. EAGLE3 speculative decoding (3 draft tokens) on both roles.
set -u
export DYN_REQUEST_PLANE=tcp
export DYN_EVENT_PLANE=zmq
export UCX_TLS=cuda_ipc,cuda_copy,tcp,self
export DYN_REASONING_PARSER=gpt_oss
export DYN_TOOL_CALL_PARSER=harmony
export ETCD_ENDPOINTS=http://localhost:2379
MODEL_PATH=openai/gpt-oss-120b
SERVED=gpt-oss-120b

etcd --data-dir /tmp/etcd --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://localhost:2379 &
sleep 4

# Prefill worker (GPU0, TP1)
CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.trtllm \
  --model-path "$MODEL_PATH" --served-model-name "$SERVED" \
  --disaggregation-mode prefill --extra-engine-args /work/nsc_prefill_engine.yaml \
  --request-plane tcp --event-plane zmq &

# Decode worker (GPU1..4, TP4 via mpirun)
CUDA_VISIBLE_DEVICES=1,2,3,4 mpirun --allow-run-as-root -n 4 trtllm-llmapi-launch \
  python3 -m dynamo.trtllm \
  --model-path "$MODEL_PATH" --served-model-name "$SERVED" \
  --disaggregation-mode decode --extra-engine-args /work/nsc_decode_engine_tp4.yaml \
  --request-plane tcp --event-plane zmq &

# Dynamo frontend (KV router, :8000)
python3 -m dynamo.frontend --http-port 8000 --request-plane tcp --event-plane zmq \
  --router-mode kv --no-router-kv-events --router-ttl-secs 2400.0 &

# gpt-oss-120b (FP4) on B200 — TensorRT-LLM 1.3.0rc12.post1, single GPU (TP1),
# Eagle3-v3 speculative decoding (3 draft tokens), stream_interval 10.
docker run -d --gpus '"device=0"' --ipc=host --shm-size=64g -p 8000:8000 \
  -v <host-path>/hf-cache:/root/.cache/huggingface \
  -v <host-path>/extra_llm_api_options.yaml:/etc/trtllm/extra_llm_api_options.yaml:ro \
  -e HF_HOME=/root/.cache/huggingface -e TRTLLM_ENABLE_PDL=1 -e OPAL_PREFIX=/opt/hpcx/ompi \
  -e OMPI_MCA_coll_hcoll_enable=0 -e UCC_CL_BASIC_TLS='^sharp' -e UCC_EC_CUDA_EXEC_NUM_THREADS=256 \
  <trtllm-image> \
  trtllm-serve openai/gpt-oss-120b --host 0.0.0.0 --port 8000 \
    --extra_llm_api_options /etc/trtllm/extra_llm_api_options.yaml

# ---------------- extra_llm_api_options.yaml ----------------
tensor_parallel_size: 1
moe_expert_parallel_size: 1
pipeline_parallel_size: 1
enable_attention_dp: false
cuda_graph_config:
  enable_padding: true
  max_batch_size: 256
kv_cache_config:
  dtype: fp8
  enable_block_reuse: true
  free_gpu_memory_fraction: 0.9
moe_config:
  backend: TRTLLM
max_seq_len: 131072
max_batch_size: 256
max_num_tokens: 20000
num_postprocess_workers: 4
stream_interval: 10
print_iter_log: true
enable_chunked_prefill: true
trust_remote_code: true
speculative_config:
  decoding_type: Eagle3
  max_draft_len: 3
  speculative_model: nvidia/gpt-oss-120b-Eagle3-v3
  eagle3_one_model: true
  eagle3_layers_to_capture: [23, 29, 35]
  allow_advanced_sampling: true