diff --git a/kiwix_download.sh b/kiwix_download.sh index 71f4570..52e6b40 100644 --- a/kiwix_download.sh +++ b/kiwix_download.sh @@ -86,10 +86,22 @@ WIKNEWS=$(latest_zim "$MIRROR/wikinews" "wikinews_en_all_nopic") inf "Checking Vikidia (kids K-8)..." VIKIDIA=$(latest_zim "$MIRROR/vikidia" "vikidia_en_all_nopic") -# Stack Overflow: domain-style filename in stack_exchange/ +# Stack Exchange sites: domain-style filenames in stack_exchange/ inf "Checking Stack Overflow..." SO=$(latest_zim "$MIRROR/stack_exchange" "stackoverflow.com_en_all") +inf "Checking Ask Ubuntu..." +ASKUBUNTU=$(latest_zim "$MIRROR/stack_exchange" "askubuntu.com_en_all") + +inf "Checking Super User..." +SUPERUSER=$(latest_zim "$MIRROR/stack_exchange" "superuser.com_en_all") + +inf "Checking Unix & Linux SE..." +UNIX=$(latest_zim "$MIRROR/stack_exchange" "unix.stackexchange.com_en_all") + +inf "Checking Server Fault..." +SERVERFAULT=$(latest_zim "$MIRROR/stack_exchange" "serverfault.com_en_all") + # Arch Wiki: _maxi is part of the base name, not the date stamp inf "Checking Arch Wiki..." ARCH=$(latest_zim "$MIRROR/other" "archlinux_en_all_maxi") @@ -135,6 +147,10 @@ echo " Wikiversity ${WIKUNI:-NOT FOUND} ~500MB" echo " WikiNews ${WIKNEWS:-NOT FOUND} ~300MB" echo " Vikidia (kids K-8) ${VIKIDIA:-NOT FOUND} ~66MB" echo " Stack Overflow ${SO:-NOT FOUND} ~3GB" +echo " Ask Ubuntu ${ASKUBUNTU:-NOT FOUND} ~1GB" +echo " Super User ${SUPERUSER:-NOT FOUND} ~1GB" +echo " Unix & Linux SE ${UNIX:-NOT FOUND} ~500MB" +echo " Server Fault ${SERVERFAULT:-NOT FOUND} ~500MB" echo " Arch Linux Wiki ${ARCH:-NOT FOUND} ~30MB" echo " TED Talks ${TED:-NOT FOUND} ~5GB" echo " PhET Simulations ${PHET:-NOT FOUND} ~500MB" @@ -144,7 +160,7 @@ echo " iFixit (repair guides) ${IFIX:-NOT FOUND} ~2GB" echo " LibreTexts (textbooks) ${LIBRE:-NOT FOUND} ~varies" echo " Project Gutenberg ${GUT:-NOT FOUND} ~60GB" echo "" -echo -e "${Y} Total estimate: ~130GB — make sure you have space!${N}" +echo -e "${Y} Total estimate: ~133GB — make sure you have space!${N}" echo "" df -h "$KIWIX_DIR" | tail -1 | awk '{print " Available disk space: " $4}' @@ -166,7 +182,11 @@ rm -f "$KIWIX_DIR/.download_pids" [[ -n "$WIKUNI" ]] && download_zim "wikiversity" "$WIKUNI" "Wikiversity" "500MB" [[ -n "$WIKNEWS" ]] && download_zim "wikinews" "$WIKNEWS" "WikiNews" "300MB" [[ -n "$VIKIDIA" ]] && download_zim "vikidia" "$VIKIDIA" "Vikidia (kids K-8)" "66MB" -[[ -n "$SO" ]] && download_zim "stack_exchange" "$SO" "Stack Overflow" "3GB" +[[ -n "$SO" ]] && download_zim "stack_exchange" "$SO" "Stack Overflow" "3GB" +[[ -n "$ASKUBUNTU" ]] && download_zim "stack_exchange" "$ASKUBUNTU" "Ask Ubuntu" "1GB" +[[ -n "$SUPERUSER" ]] && download_zim "stack_exchange" "$SUPERUSER" "Super User" "1GB" +[[ -n "$UNIX" ]] && download_zim "stack_exchange" "$UNIX" "Unix & Linux SE" "500MB" +[[ -n "$SERVERFAULT" ]] && download_zim "stack_exchange" "$SERVERFAULT" "Server Fault" "500MB" [[ -n "$ARCH" ]] && download_zim "other" "$ARCH" "Arch Linux Wiki" "30MB" [[ -n "$TED" ]] && download_zim "ted" "$TED" "TED Talks" "5GB" [[ -n "$PHET" ]] && download_zim "phet" "$PHET" "PhET Simulations" "500MB" diff --git a/kiwix_update.sh b/kiwix_update.sh new file mode 100755 index 0000000..94e04c7 --- /dev/null +++ b/kiwix_update.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# ============================================================================= +# Kiwix ZIM Auto-Updater +# Checks each ZIM for a newer version on the mirror, downloads it, removes old. +# Safe to run as a cron job — only acts when a newer version exists. +# Usage: bash kiwix_update.sh +# Cron example (monthly): 0 3 1 * * /bin/bash /path/to/kiwix_update.sh +# ============================================================================= + +KIWIX_DIR="$HOME/docker/ai-stack/kiwix" +MIRROR="https://ftp.fau.de/kiwix/zim" +MIRROR2="https://download.kiwix.org/zim" +LOG="$HOME/docker/ai-stack/logs/kiwix-update.log" +mkdir -p "$KIWIX_DIR" "$(dirname "$LOG")" + +G='\033[0;32m'; Y='\033[1;33m'; C='\033[0;36m'; R='\033[0;31m'; N='\033[0m' +ok() { echo -e "${G}[OK]${N} $1" | tee -a "$LOG"; } +inf() { echo -e "${C}[..]${N} $1" | tee -a "$LOG"; } +wrn() { echo -e "${Y}[!!]${N} $1" | tee -a "$LOG"; } +err() { echo -e "${R}[XX]${N} $1" | tee -a "$LOG"; } + +echo "" | tee -a "$LOG" +echo "=== Kiwix Update Check: $(date) ===" | tee -a "$LOG" +echo "" | tee -a "$LOG" + +# Find latest filename on mirror +latest_zim() { + local base_url="$1" + local pattern="$2" + curl -s "$base_url/" | \ + grep -oP "${pattern}_[\d-]+\.zim" | \ + sort -u | tail -1 +} + +# Find what's currently on disk matching a pattern +current_zim() { + local pattern="$1" + ls "$KIWIX_DIR"/${pattern}_*.zim 2>/dev/null | sort | tail -1 | xargs basename 2>/dev/null +} + +# Check and update one ZIM +# Usage: check_update "category" "pattern" "description" ["mirror"] +UPDATED=0 +check_update() { + local category="$1" + local pattern="$2" + local description="$3" + local mirror="${4:-$MIRROR}" + + inf "Checking $description..." + local latest + latest=$(latest_zim "$mirror/$category" "$pattern") + + if [[ -z "$latest" ]]; then + wrn "$description: could not find latest on mirror — skipping" + return + fi + + local current + current=$(current_zim "$pattern") + + if [[ -z "$current" ]]; then + inf "$description: not downloaded yet — skipping (run kiwix_download.sh first)" + return + fi + + if [[ "$latest" == "$current" ]]; then + ok "$description: up to date ($current)" + return + fi + + echo "" + inf "$description: update available" + inf " Current: $current" + inf " Latest: $latest" + + local url="$mirror/$category/$latest" + local dest="$KIWIX_DIR/$latest" + local old="$KIWIX_DIR/$current" + + inf " Downloading $latest..." + if wget -q --show-progress -c "$url" -O "$dest" >> "$LOG" 2>&1; then + ok " Download complete — removing old file" + rm -f "$old" + UPDATED=$((UPDATED + 1)) + else + err " Download failed — keeping old file" + rm -f "$dest" + fi + echo "" +} + +# ── Check each ZIM ───────────────────────────────────────────────────────── +check_update "wikipedia" "wikipedia_en_all_nopic" "Wikipedia" +check_update "wiktionary" "wiktionary_en_all_nopic" "Wiktionary" +check_update "wikiquote" "wikiquote_en_all_nopic" "Wikiquote" +check_update "wikisource" "wikisource_en_all_nopic" "Wikisource" +check_update "wikibooks" "wikibooks_en_all_nopic" "Wikibooks" +check_update "wikivoyage" "wikivoyage_en_all_nopic" "Wikivoyage" +check_update "wikiversity" "wikiversity_en_all_nopic" "Wikiversity" +check_update "wikinews" "wikinews_en_all_nopic" "WikiNews" +check_update "vikidia" "vikidia_en_all_nopic" "Vikidia" +check_update "stack_exchange" "stackoverflow.com_en_all" "Stack Overflow" +check_update "stack_exchange" "askubuntu.com_en_all" "Ask Ubuntu" +check_update "stack_exchange" "superuser.com_en_all" "Super User" +check_update "stack_exchange" "unix.stackexchange.com_en_all" "Unix & Linux SE" +check_update "stack_exchange" "serverfault.com_en_all" "Server Fault" +check_update "other" "archlinux_en_all_maxi" "Arch Wiki" +check_update "ted" "ted_mul_youth" "TED Talks" +check_update "phet" "phet_en_all" "PhET Simulations" +check_update "devdocs" "devdocs_en_zig" "DevDocs" +check_update "freecodecamp" "freecodecamp_en_all" "FreeCodeCamp" +check_update "ifixit" "ifixit_en_all" "iFixit" +check_update "libretexts" "libretexts.org_en_workforce" "LibreTexts" +check_update "gutenberg" "gutenberg_en_all" "Project Gutenberg" "$MIRROR2" + +# ── Restart Kiwix if anything changed ───────────────────────────────────── +echo "" +if [[ $UPDATED -gt 0 ]]; then + ok "$UPDATED ZIM(s) updated — restarting Kiwix container" + docker compose -f "$HOME/docker/ai-stack/docker-compose.yml" restart kiwix >> "$LOG" 2>&1 \ + && ok "Kiwix restarted" \ + || wrn "Could not restart Kiwix — do it manually: docker compose restart kiwix" +else + ok "All ZIMs are up to date — no restart needed" +fi + +echo "" +echo "=== Update check complete: $(date) ===" | tee -a "$LOG" +echo "" diff --git a/local-ai-setup.sh b/local-ai-setup.sh index 2ac56bc..04d8cf8 100755 --- a/local-ai-setup.sh +++ b/local-ai-setup.sh @@ -513,7 +513,7 @@ cat > "$BASE/docker-compose.yml" << COMPOSE # Pull latest images: docker compose pull && docker compose up -d # Show status: docker compose ps # -# Services: ollama open-webui chromadb rag-server mcp-server +# Services: ollama open-webui chromadb rag-server mcp-server aider # kiwix gitea invokeai portainer # ─────────────────────────────────────────────────────────────────────────────── @@ -612,11 +612,33 @@ services: - REPOS_DIR=/repos - GITEA_URL=http://gitea:3000 - RAG_URL=http://rag-server:8001 + - KIWIX_URL=http://kiwix:80 command: > bash -c "apt-get update -qq && apt-get install -y --no-install-recommends git ripgrep && pip install --no-cache-dir -r mcp_requirements.txt && python mcp_server.py" - depends_on: [rag-server] + depends_on: [rag-server, kiwix] + + aider: + image: paulgauthier/aider:latest + container_name: aider + restart: unless-stopped + ports: ["0.0.0.0:8080:8501"] + volumes: + - $BASE/workspace:/workspace + - $BASE/repos:/repos + working_dir: /workspace + environment: + - OLLAMA_API_BASE=http://ollama:11434 + - GIT_AUTHOR_NAME=aider + - GIT_AUTHOR_EMAIL=aider@local + - GIT_COMMITTER_NAME=aider + - GIT_COMMITTER_EMAIL=aider@local + command: > + --gui --no-auto-commits --no-check-update + --model ollama/$CODE_MODEL + depends_on: + ollama: {condition: service_healthy} kiwix: image: ghcr.io/kiwix/kiwix-serve:latest @@ -687,7 +709,7 @@ if command -v ufw &>/dev/null && [[ ! -f "$BASE/.ufw-done" ]] || $FORCE; then read -rp " LAN subnet [192.168.1.0/24]: " LAN; LAN="${LAN:-192.168.1.0/24}" [[ "$LAN" =~ /[0-9]+$ ]] || LAN="${LAN}/24" for pc in "3000:Open WebUI" "11434:Ollama" "8001:RAG" "8002:MCP" \ - "8000:ChromaDB" "8181:Kiwix" \ + "8000:ChromaDB" "8080:Aider" "8181:Kiwix" \ "3001:Gitea" "2222:Gitea SSH" "9090:InvokeAI" \ "9000:Portainer" "9443:Portainer S"; do sudo ufw allow from "$LAN" to any port "${pc%%:*}" proto tcp comment "${pc##*:}" >/dev/null @@ -705,6 +727,7 @@ docker compose pull --quiet 2>/dev/null docker compose up -d echo "" echo " Open WebUI → http://$LOCAL_IP:3000" +echo " Aider UI → http://$LOCAL_IP:8080 (Claude Code-like editor)" echo " InvokeAI → http://$LOCAL_IP:9090" echo " Kiwix → http://$LOCAL_IP:8181 (run kiwix_download.sh first)" echo " Gitea → http://$LOCAL_IP:3001" @@ -758,6 +781,39 @@ PULLSH chmod +x "$BASE/pull-models.sh" ok "start.sh stop.sh status.sh pull-models.sh" +cat > "$BASE/aider.sh" << AIDSH +#!/bin/bash +# Aider CLI — Claude Code-like terminal experience against any local git repo. +# Usage: +# ./aider.sh # interactive, files from stdin +# ./aider.sh src/main.py # open specific files +# ./aider.sh --model ollama/qwen2.5-coder:7b src/foo.py # override model +# +# Tip: clone Gitea repos into $BASE/repos/, then: +# cd $BASE/repos/my-project && $BASE/aider.sh + +MODEL="\${AIDER_MODEL:-ollama/$CODE_MODEL}" +REPO_ROOT="\$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + +docker run --rm -it \\ + --network local-ai_default \\ + -v "\$REPO_ROOT:\$REPO_ROOT" \\ + -v "$BASE/repos:/repos" \\ + -w "\$REPO_ROOT" \\ + -e OLLAMA_API_BASE=http://ollama:11434 \\ + -e GIT_AUTHOR_NAME=aider \\ + -e GIT_AUTHOR_EMAIL=aider@local \\ + -e GIT_COMMITTER_NAME=aider \\ + -e GIT_COMMITTER_EMAIL=aider@local \\ + paulgauthier/aider:latest \\ + --no-auto-commits \\ + --no-check-update \\ + --model "\$MODEL" \\ + "\$@" +AIDSH +chmod +x "$BASE/aider.sh" +ok "aider.sh" + # ============================================================================= section "Systemd" # ============================================================================= @@ -804,6 +860,7 @@ echo -e "${GREEN}${BOLD} Done! GPU: ${VRAM_GB}GB → $TIER${NC}" echo -e "${GREEN}${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e " ${CYAN}Open WebUI${NC} → http://$LOCAL_IP:3000" +echo -e " ${CYAN}Aider UI${NC} → http://$LOCAL_IP:8080 (browser coding assistant)" echo -e " ${CYAN}InvokeAI${NC} → http://$LOCAL_IP:9090" echo -e " ${CYAN}Kiwix${NC} → http://$LOCAL_IP:8181" echo -e " ${CYAN}Gitea${NC} → http://$LOCAL_IP:3001" @@ -819,6 +876,9 @@ echo -e " ${YELLOW}PDFs:${NC} $BASE/papers/" echo -e " ${YELLOW}Workspace:${NC} $BASE/workspace/" echo -e " ${YELLOW}ZIMs:${NC} ./kiwix_download.sh" echo "" +echo -e " ${YELLOW}Aider CLI:${NC} cd your-repo && $BASE/aider.sh " +echo " (or open browser UI above — both use your local code model)" +echo "" echo -e " ${YELLOW}Save Claude usage:${NC} use local 14B for boilerplate, docs," echo " simple fixes. Use Claude Sonnet 4.6 for hard bugs," echo " multi-file refactoring, architecture decisions." diff --git a/mcp_server.py b/mcp_server.py index 7003ac9..65bec0a 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -16,6 +16,7 @@ GITEA_URL = os.getenv("GITEA_URL", "http://gitea:3000") GITEA_TOKEN = os.getenv("GITEA_TOKEN", "") GITHUB_TOKEN= os.getenv("GITHUB_TOKEN","") RAG_URL = os.getenv("RAG_URL", "http://rag-server:8001") +KIWIX_URL = os.getenv("KIWIX_URL", "http://kiwix:80") mcp = FastMCP("local-dev-tools") @@ -180,6 +181,27 @@ def github_api(method: str, endpoint: str, body: str = "") -> str: except Exception: return r.text +# ── Kiwix search ────────────────────────────────────────────────────────────── +import re as _re + +@mcp.tool() +def kiwix_search(query: str, books: str = "") -> str: + """Search offline ZIM content via Kiwix (Wikipedia, Stack Overflow, etc). + books: optional comma-separated book names to filter, e.g. 'stackoverflow.com_en_all'. + Leave empty to search all ZIMs.""" + params = {"pattern": query, "lang": ""} + if books: + params["books"] = books + try: + r = httpx.get(f"{KIWIX_URL}/search", params=params, timeout=15) + # strip HTML tags to get plain text + text = _re.sub(r"<[^>]+>", " ", r.text) + text = _re.sub(r"&[a-zA-Z]+;", " ", text) + text = _re.sub(r"\s{2,}", "\n", text).strip() + return text[:6000] or "(no results)" + except Exception as e: + return f"[kiwix error] {e}" + # ── RAG ingest ──────────────────────────────────────────────────────────────── @mcp.tool() def ingest_repo(url: str, name: str = "", branch: str = "main") -> str: diff --git a/ollama_update.sh b/ollama_update.sh new file mode 100755 index 0000000..3173f92 --- /dev/null +++ b/ollama_update.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# ============================================================================= +# Ollama Model Auto-Updater +# Pulls latest version of every installed model. Ollama compares digests +# server-side — no download happens if the model is already current. +# Safe to run as a systemd timer. +# ============================================================================= + +OLLAMA_URL="${OLLAMA_HOST:-http://localhost:11434}" +LOG="$HOME/docker/ai-stack/logs/ollama-update.log" +mkdir -p "$(dirname "$LOG")" + +G='\033[0;32m'; Y='\033[1;33m'; C='\033[0;36m'; R='\033[0;31m'; N='\033[0m' +ok() { echo -e "${G}[OK]${N} $1" | tee -a "$LOG"; } +inf() { echo -e "${C}[..]${N} $1" | tee -a "$LOG"; } +wrn() { echo -e "${Y}[!!]${N} $1" | tee -a "$LOG"; } +err() { echo -e "${R}[XX]${N} $1" | tee -a "$LOG"; } + +echo "" | tee -a "$LOG" +echo "=== Ollama Model Update: $(date) ===" | tee -a "$LOG" +echo "" | tee -a "$LOG" + +# Check Ollama is reachable +if ! curl -sf "$OLLAMA_URL/api/tags" > /dev/null; then + err "Ollama not reachable at $OLLAMA_URL — is the container running?" + exit 1 +fi + +# Get installed model names from the API +mapfile -t MODELS < <( + curl -sf "$OLLAMA_URL/api/tags" | \ + grep -oP '"name"\s*:\s*"\K[^"]+' | \ + sort -u +) + +if [[ ${#MODELS[@]} -eq 0 ]]; then + wrn "No models found — nothing to update" + exit 0 +fi + +inf "Found ${#MODELS[@]} model(s): ${MODELS[*]}" +echo "" | tee -a "$LOG" + +UPDATED=0 +FAILED=0 + +for model in "${MODELS[@]}"; do + inf "Pulling $model..." + output=$(ollama pull "$model" 2>&1) + exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + err "$model: pull failed" + echo "$output" >> "$LOG" + FAILED=$((FAILED + 1)) + elif echo "$output" | grep -q "up to date"; then + ok "$model: already up to date" + else + ok "$model: updated" + UPDATED=$((UPDATED + 1)) + fi +done + +echo "" | tee -a "$LOG" +inf "Done — $UPDATED updated, $FAILED failed" +echo "=== Complete: $(date) ===" | tee -a "$LOG" +echo "" | tee -a "$LOG" diff --git a/systemd/install.sh b/systemd/install.sh new file mode 100755 index 0000000..4f8c2bd --- /dev/null +++ b/systemd/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Install all local-ai systemd user units and enable timers +set -e + +UNIT_DIR="$HOME/.config/systemd/user" +SRC="$(cd "$(dirname "$0")" && pwd)" + +mkdir -p "$UNIT_DIR" + +for unit in "$SRC"/*.service "$SRC"/*.timer; do + name=$(basename "$unit") + ln -sf "$unit" "$UNIT_DIR/$name" + echo "Linked $name" +done + +systemctl --user daemon-reload + +for timer in "$SRC"/*.timer; do + name=$(basename "$timer") + systemctl --user enable --now "$name" + echo "Enabled $name" +done + +echo "" +systemctl --user list-timers kiwix-update.timer ollama-update.timer +echo "" +echo "Run manually:" +echo " systemctl --user start kiwix-update.service" +echo " systemctl --user start ollama-update.service" +echo "" +echo "Logs:" +echo " journalctl --user -u kiwix-update.service -f" +echo " journalctl --user -u ollama-update.service -f" diff --git a/systemd/kiwix-update.service b/systemd/kiwix-update.service new file mode 100644 index 0000000..3ac3091 --- /dev/null +++ b/systemd/kiwix-update.service @@ -0,0 +1,10 @@ +[Unit] +Description=Kiwix ZIM auto-updater +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/bin/bash %h/local-ai/kiwix_update.sh +StandardOutput=journal +StandardError=journal diff --git a/systemd/kiwix-update.timer b/systemd/kiwix-update.timer new file mode 100644 index 0000000..05df2d5 --- /dev/null +++ b/systemd/kiwix-update.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Monthly Kiwix ZIM update check +Requires=kiwix-update.service + +[Timer] +OnCalendar=monthly +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/ollama-update.service b/systemd/ollama-update.service new file mode 100644 index 0000000..25a976b --- /dev/null +++ b/systemd/ollama-update.service @@ -0,0 +1,10 @@ +[Unit] +Description=Ollama model auto-updater +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/bin/bash %h/local-ai/ollama_update.sh +StandardOutput=journal +StandardError=journal diff --git a/systemd/ollama-update.timer b/systemd/ollama-update.timer new file mode 100644 index 0000000..9051082 --- /dev/null +++ b/systemd/ollama-update.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Monthly Ollama model update check +Requires=ollama-update.service + +[Timer] +OnCalendar=monthly +RandomizedDelaySec=2h +Persistent=true + +[Install] +WantedBy=timers.target