Gan, Jimmy
3994e29cb0
Full documentation of optimizing two LLMs on a single M5 Max GPU: - KV cache quantization (Q4_0) - Flash attention and batch tuning - Router mode with --models-max 1 - Per-model thread optimization via INI presets - Before/after benchmarks (12→48 t/s on 27B, 23→132 t/s on 35B)
40 lines
1.5 KiB
Plaintext
40 lines
1.5 KiB
Plaintext
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0">
|
|
<dict>
|
|
<key>KeepAlive</key><true/>
|
|
<key>Label</key><string>com.example.llama-server-router</string>
|
|
<key>ProgramArguments</key>
|
|
<array>
|
|
<string>/opt/homebrew/bin/llama-server</string>
|
|
<string>--models-dir</string>
|
|
<string>/Users/username/.hermes/models-router</string>
|
|
<string>--models-max</string><string>1</string>
|
|
<string>--models-preset</string>
|
|
<string>/Users/username/.hermes/llama-models.ini</string>
|
|
<string>--host</string><string>0.0.0.0</string>
|
|
<string>--port</string><string>8085</string>
|
|
<string>-ngl</string><string>99</string>
|
|
<string>-c</string><string>131072</string>
|
|
<string>--mlock</string>
|
|
<string>--spec-type</string><string>draft-mtp</string>
|
|
<string>--spec-draft-n-max</string><string>3</string>
|
|
<string>--batch-size</string><string>512</string>
|
|
<string>--ubatch-size</string><string>128</string>
|
|
<string>--cache-type-k</string><string>q4_0</string>
|
|
<string>--cache-type-v</string><string>q4_0</string>
|
|
<string>--flash-attn</string><string>on</string>
|
|
<string>--metrics</string>
|
|
</array>
|
|
<key>RunAtLoad</key><true/>
|
|
<key>StandardOutPath</key>
|
|
<string>/tmp/llama-server-router.log</string>
|
|
<key>StandardErrorPath</key>
|
|
<string>/tmp/llama-server-router.log</string>
|
|
<key>ThrottleInterval</key>
|
|
<integer>5</integer>
|
|
<key>WorkingDirectory</key>
|
|
<string>/Users/username</string>
|
|
</dict>
|
|
</plist>
|