# Model: https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q5_K_S.gguf
# normal: 121600, pipeline off: 195584, MTP RS 224256 (with desktop SW rendering), 240640 with vulkan 60% TG

# ROCm ctx VEC forced: 224256 q8_0, TG speed: 87.15 t/s, PP for 32k:  557.50 t/s, 96k: 244.44 t/s
# ROCm ctx VEC off   : 163072 q8_0, TG speed: 84.84 t/s, PP for 32k: 1216.19 t/s, 96k: 779.30 t/s
# Vulkan: ctx:         229376 q8_0, TG speed: 63.82 t/s, PP for 32k:  684.04 t/s, 96k: 426.5


# 1. Set Environment Variables
export LD_LIBRARY_PATH="/home/eaman/llama/bin_vulkan"
# ROCm 7.14 can segfault in hipGraphExecUpdate during long MTP runs.
# it's actually faster without it!
# Note: -ot and fine allocation of layer is now auto, no need to split-layers and --pipeline-parallel off
# but keep --pipeline-parallel off as redundancy in case there's no need to fine fitting
export GGML_CUDA_DISABLE_GRAPHS=1
# Keep graphs enabled globally; disable them only for this MoE launcher.
#	--no-spec-draft-backend-sampling \
# It makes the entire MTP sampling chain CPU-side because with this QWEN
# CUB on ROCm is up to only 1,024 elements, reduces synch and transfer costs.



# 2. Run the Server
llama-server  -m unsloth/A3B/Qwen3.6-35B-A3B-UD-Q5_K_S.gguf \
        --host 0.0.0.0  -fa on --load-mode none --jinja \
        -ctk q8_0 -ctv q8_0 \
        -b 2250 -ub 750 \
        --temp 0.8  --top-k 20 --top-p 0.95 --min-p 0.0 \
        --repeat-penalty 1.0 --presence_penalty 0.0 \
        --spec-mtp-cr-depth 1 --pipeline-parallel off --spec-draft-adaptive \
        --spec-type draft-mtp,ngram-mod --spec-draft-p-min 0.85 --spec-draft-n-max 5 \
        --spec-ngram-mod-n-match 24 --spec-ngram-mod-n-min 8 --spec-ngram-mod-n-max 32 \
        -ctkd q4_0 -ctvd q4_0 \
        --reasoning on --reasoning-preserve --reasoning-budget 6000  --reasoning-budget-message " -- Reasoning budget exceeded, proceed to final answer."  \
        --ctx-checkpoints 256 -lv 3 --no-warmup --cache-ram 6000 -np 1  --fit-target 70 --no-log-timestamps \
        --device vulkan0,vulkan1 --split-mode layer  --tensor-split 0.59,0.41   -c 229376