Add context awareness: token tracker + compaction + 6GB GPU tier
Context management for local models with limited windows: - Context Tracker (community function): shows tokens used vs available, progress bar, percentage remaining — so you see the cliff coming - Checkpoint Summarization Filter: auto-summarizes old messages when context fills up, like Claude's auto-compaction - Both added as recommended post-install links in setup output (Open WebUI Functions install with one click from the UI) 6GB GPU tier (Quadro P3300, GTX 1060, etc.): - Qwen 3.5 4B at Q4_K_M = ~2.5GB weights, leaves 3.5GB for KV cache - With Q8 KV cache: ~32K usable context on 6GB - Better than squeezing 9B into nothing — more context > slightly smarter https://claude.ai/code/session_01PtYTPherSJaxDEVPgF6Nxu
This commit is contained in:
@@ -66,6 +66,9 @@ elif [[ "$TOTAL_VRAM" -ge 14 ]]; then
|
|||||||
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
||||||
CTX=32768; GPU_TIER="${TOTAL_VRAM}GB VRAM — 9B dense, 32K context"
|
CTX=32768; GPU_TIER="${TOTAL_VRAM}GB VRAM — 9B dense, 32K context"
|
||||||
|
elif [[ "$TOTAL_VRAM" -ge 6 ]]; then
|
||||||
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
|
CTX=32768; GPU_TIER="${TOTAL_VRAM}GB VRAM — 4B + KV quant, 32K context (3.5GB free for cache)"
|
||||||
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
CTX=16384; GPU_TIER="${TOTAL_VRAM}GB VRAM — 4B models, 16K context"
|
CTX=16384; GPU_TIER="${TOTAL_VRAM}GB VRAM — 4B models, 16K context"
|
||||||
@@ -1391,6 +1394,14 @@ if $INSTALL_AI; then
|
|||||||
echo " Via MCP: gitea_github_sync(mode='all')"
|
echo " Via MCP: gitea_github_sync(mode='all')"
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
|
if $INSTALL_AI; then
|
||||||
|
echo ""
|
||||||
|
echo -e " ${YELLOW}Recommended Open WebUI Functions (install from Admin → Functions → +):${NC}"
|
||||||
|
echo " Context tracker: https://openwebui.com/f/centrisic/context_tracker"
|
||||||
|
echo " → Shows tokens used vs available, progress bar, context % remaining"
|
||||||
|
echo " Context compaction: https://openwebui.com/f/projectmoon/checkpoint_summarization_filter"
|
||||||
|
echo " → Auto-summarizes old messages when context fills up (like Claude)"
|
||||||
|
fi
|
||||||
if $SVC_KIWIX && [[ "$ZIM_CHOICE" == "3" ]]; then
|
if $SVC_KIWIX && [[ "$ZIM_CHOICE" == "3" ]]; then
|
||||||
echo -e " ${YELLOW}ZIM downloads:${NC} ./kiwix_download.sh (not started)"
|
echo -e " ${YELLOW}ZIM downloads:${NC} ./kiwix_download.sh (not started)"
|
||||||
elif $SVC_KIWIX; then
|
elif $SVC_KIWIX; then
|
||||||
|
|||||||
+11
-2
@@ -42,6 +42,9 @@ elif [[ "$TOTAL_VRAM" -ge 14 ]]; then
|
|||||||
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
||||||
CTX=32768; TIER="${TOTAL_VRAM}GB — 9B dense, 32K context"
|
CTX=32768; TIER="${TOTAL_VRAM}GB — 9B dense, 32K context"
|
||||||
|
elif [[ "$TOTAL_VRAM" -ge 6 ]]; then
|
||||||
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
|
CTX=32768; TIER="${TOTAL_VRAM}GB — 4B + KV quant, 32K context"
|
||||||
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
CTX=16384; TIER="${TOTAL_VRAM}GB — 4B models, 16K context"
|
CTX=16384; TIER="${TOTAL_VRAM}GB — 4B models, 16K context"
|
||||||
@@ -898,7 +901,13 @@ echo ""
|
|||||||
echo -e " ${YELLOW}Aider CLI:${NC} cd your-repo && $BASE/aider.sh <files>"
|
echo -e " ${YELLOW}Aider CLI:${NC} cd your-repo && $BASE/aider.sh <files>"
|
||||||
echo " (or open browser UI above — both use your local code model)"
|
echo " (or open browser UI above — both use your local code model)"
|
||||||
echo ""
|
echo ""
|
||||||
echo -e " ${YELLOW}Save Claude usage:${NC} use local 14B for boilerplate, docs,"
|
echo -e " ${YELLOW}Recommended Open WebUI Functions${NC} (install from Admin → Functions → +):"
|
||||||
echo " simple fixes. Use Claude Sonnet 4.6 for hard bugs,"
|
echo " Context tracker: https://openwebui.com/f/centrisic/context_tracker"
|
||||||
|
echo " → Shows tokens used vs available, progress bar, context % remaining"
|
||||||
|
echo " Context compaction: https://openwebui.com/f/projectmoon/checkpoint_summarization_filter"
|
||||||
|
echo " → Auto-summarizes old messages when context fills up (like Claude)"
|
||||||
|
echo ""
|
||||||
|
echo -e " ${YELLOW}Save Claude usage:${NC} use local models for boilerplate, docs,"
|
||||||
|
echo " simple fixes. Use Claude for hard bugs,"
|
||||||
echo " multi-file refactoring, architecture decisions."
|
echo " multi-file refactoring, architecture decisions."
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
@@ -1125,6 +1125,9 @@ elif [[ "$TOTAL_VRAM" -ge 14 ]]; then
|
|||||||
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 8 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
CHAT_MODEL="qwen3.5:9b"; CODE_MODEL="qwen3.5:9b"
|
||||||
CTX=32768; TIER="${TOTAL_VRAM}GB — 9B dense, 32K context"
|
CTX=32768; TIER="${TOTAL_VRAM}GB — 9B dense, 32K context"
|
||||||
|
elif [[ "$TOTAL_VRAM" -ge 6 ]]; then
|
||||||
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
|
CTX=32768; TIER="${TOTAL_VRAM}GB — 4B + KV quant, 32K context"
|
||||||
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
elif [[ "$TOTAL_VRAM" -ge 4 ]]; then
|
||||||
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
CHAT_MODEL="qwen3.5:4b"; CODE_MODEL="qwen3.5:4b"
|
||||||
CTX=16384; TIER="${TOTAL_VRAM}GB — 4B models, 16K context"
|
CTX=16384; TIER="${TOTAL_VRAM}GB — 4B models, 16K context"
|
||||||
|
|||||||
Reference in New Issue
Block a user