# Model: https://huggingface.co/bartowski/Qwen_Qwen3.6-27B-GGUF?show_file_info=Qwen_Qwen3.6-27B-Q6_K_L.gguf
# Setup AMD 6800 16GB + 7600xt 12GB ROCm layer split
# CTX 119,040 normal MTP n=5, 149,504 MTP CR, 136,448 normal n=3 (same speed as MTP CR n=5).

# ROCm max speed: 30.29 tokens per second with draft acceptance = 0.98186 for HTML / coding
# Ctx:  q8/q8_0 136448 ctx, q8_0 /q5_1  159232 ctx VEC oprimized, 147712 not -ot optimized (1t/s faster),
# Mainline llama.cp not pathched: 50432 ( no -ot possible)

# ROCm ctx VEC forced: 148736 q8_0, TG speed: 27.13 t/s, PP for 32k: 183.22 t/s, 96k: 95.56
# ROCm ctx VEC off   : 101632 q8_0, TG speed: 27.39 t/s, PP for 32k: 298.55 t/s, 96k: 232.41 t/s
# Vulkan: ctx:         144384 q8_0, TG speed: 24.65 t/s, PP for 32k: 201.14 t/s, 96k: 139.81 t/s

# -ot '^blk\.39\.ffn_(up|gate|down)\.weight$=ROCm1'
# 209.18 MiB movement
# Needs to be fine tuned calculated for each model on dual gpu
# as normal --tensor-split does only a broad estimate that doesn't advantage from quantization of MTP KV cache
# Use dual_gpu_context_balancing_guide.md to do that or delete the proposed example -ot .

# Warning: remove -ot and --pipeline-parallel if you are not running patched llama.cp


# 1. Run the Server
llama-server  -m bartowski/Qwen3.8-27B-Q6_K_L.gguf \
	-fa on --load-mode none \
	-b 1024 -ub 384 --fit-target 20 \
	-ctk q8_0 -ctv q8_0 \
	--temp 0.8  --top-k 20 --top-p 0.95 --min-p 0.0 \
	--presence-penalty 0.0 --repeat-penalty 1.0 \
	--pipeline-parallel off --spec-mtp-cr-depth 1 --spec-draft-adaptive \
	--spec-type draft-mtp,ngram-mod --spec-draft-p-min 0.85 --spec-draft-n-max 6 \
	--cache-type-k-draft q4_0 --cache-type-v-draft q4_0 \
	--spec-ngram-mod-n-match 24 --spec-ngram-mod-n-min 8 --spec-ngram-mod-n-max 32 \
	--reasoning on --chat-template-kwargs '{"reasoning_effort":"medium"}' --reasoning-preserve --reasoning-budget 14000 --reasoning-budget-message " -- Reasoning budget exceeded, proceed to final answer." \
	--ctx-checkpoints 148 --no-mmproj-offload --cache-ram 8192 -np 1 -ngl 99 -lv 3 --no-warmup  --timeout 900 \
	--device rocm0,rocm1 --split-mode layer --tensor-split 0.6,0.4 -ot '^blk\.39\.ffn_(up|gate|down)\.weight$=ROCm1'