diff --git a/kiwix_download.sh b/kiwix_download.sh index 71f4570..52e6b40 100644 --- a/kiwix_download.sh +++ b/kiwix_download.sh @@ -86,10 +86,22 @@ WIKNEWS=$(latest_zim "$MIRROR/wikinews" "wikinews_en_all_nopic") inf "Checking Vikidia (kids K-8)..." VIKIDIA=$(latest_zim "$MIRROR/vikidia" "vikidia_en_all_nopic") -# Stack Overflow: domain-style filename in stack_exchange/ +# Stack Exchange sites: domain-style filenames in stack_exchange/ inf "Checking Stack Overflow..." SO=$(latest_zim "$MIRROR/stack_exchange" "stackoverflow.com_en_all") +inf "Checking Ask Ubuntu..." +ASKUBUNTU=$(latest_zim "$MIRROR/stack_exchange" "askubuntu.com_en_all") + +inf "Checking Super User..." +SUPERUSER=$(latest_zim "$MIRROR/stack_exchange" "superuser.com_en_all") + +inf "Checking Unix & Linux SE..." +UNIX=$(latest_zim "$MIRROR/stack_exchange" "unix.stackexchange.com_en_all") + +inf "Checking Server Fault..." +SERVERFAULT=$(latest_zim "$MIRROR/stack_exchange" "serverfault.com_en_all") + # Arch Wiki: _maxi is part of the base name, not the date stamp inf "Checking Arch Wiki..." ARCH=$(latest_zim "$MIRROR/other" "archlinux_en_all_maxi") @@ -135,6 +147,10 @@ echo " Wikiversity ${WIKUNI:-NOT FOUND} ~500MB" echo " WikiNews ${WIKNEWS:-NOT FOUND} ~300MB" echo " Vikidia (kids K-8) ${VIKIDIA:-NOT FOUND} ~66MB" echo " Stack Overflow ${SO:-NOT FOUND} ~3GB" +echo " Ask Ubuntu ${ASKUBUNTU:-NOT FOUND} ~1GB" +echo " Super User ${SUPERUSER:-NOT FOUND} ~1GB" +echo " Unix & Linux SE ${UNIX:-NOT FOUND} ~500MB" +echo " Server Fault ${SERVERFAULT:-NOT FOUND} ~500MB" echo " Arch Linux Wiki ${ARCH:-NOT FOUND} ~30MB" echo " TED Talks ${TED:-NOT FOUND} ~5GB" echo " PhET Simulations ${PHET:-NOT FOUND} ~500MB" @@ -144,7 +160,7 @@ echo " iFixit (repair guides) ${IFIX:-NOT FOUND} ~2GB" echo " LibreTexts (textbooks) ${LIBRE:-NOT FOUND} ~varies" echo " Project Gutenberg ${GUT:-NOT FOUND} ~60GB" echo "" -echo -e "${Y} Total estimate: ~130GB — make sure you have space!${N}" +echo -e "${Y} Total estimate: ~133GB — make sure you have space!${N}" echo "" df -h "$KIWIX_DIR" | tail -1 | awk '{print " Available disk space: " $4}' @@ -166,7 +182,11 @@ rm -f "$KIWIX_DIR/.download_pids" [[ -n "$WIKUNI" ]] && download_zim "wikiversity" "$WIKUNI" "Wikiversity" "500MB" [[ -n "$WIKNEWS" ]] && download_zim "wikinews" "$WIKNEWS" "WikiNews" "300MB" [[ -n "$VIKIDIA" ]] && download_zim "vikidia" "$VIKIDIA" "Vikidia (kids K-8)" "66MB" -[[ -n "$SO" ]] && download_zim "stack_exchange" "$SO" "Stack Overflow" "3GB" +[[ -n "$SO" ]] && download_zim "stack_exchange" "$SO" "Stack Overflow" "3GB" +[[ -n "$ASKUBUNTU" ]] && download_zim "stack_exchange" "$ASKUBUNTU" "Ask Ubuntu" "1GB" +[[ -n "$SUPERUSER" ]] && download_zim "stack_exchange" "$SUPERUSER" "Super User" "1GB" +[[ -n "$UNIX" ]] && download_zim "stack_exchange" "$UNIX" "Unix & Linux SE" "500MB" +[[ -n "$SERVERFAULT" ]] && download_zim "stack_exchange" "$SERVERFAULT" "Server Fault" "500MB" [[ -n "$ARCH" ]] && download_zim "other" "$ARCH" "Arch Linux Wiki" "30MB" [[ -n "$TED" ]] && download_zim "ted" "$TED" "TED Talks" "5GB" [[ -n "$PHET" ]] && download_zim "phet" "$PHET" "PhET Simulations" "500MB" diff --git a/local-ai-setup.sh b/local-ai-setup.sh index 2ac56bc..1bed487 100755 --- a/local-ai-setup.sh +++ b/local-ai-setup.sh @@ -612,11 +612,12 @@ services: - REPOS_DIR=/repos - GITEA_URL=http://gitea:3000 - RAG_URL=http://rag-server:8001 + - KIWIX_URL=http://kiwix:80 command: > bash -c "apt-get update -qq && apt-get install -y --no-install-recommends git ripgrep && pip install --no-cache-dir -r mcp_requirements.txt && python mcp_server.py" - depends_on: [rag-server] + depends_on: [rag-server, kiwix] kiwix: image: ghcr.io/kiwix/kiwix-serve:latest diff --git a/mcp_server.py b/mcp_server.py index 7003ac9..65bec0a 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -16,6 +16,7 @@ GITEA_URL = os.getenv("GITEA_URL", "http://gitea:3000") GITEA_TOKEN = os.getenv("GITEA_TOKEN", "") GITHUB_TOKEN= os.getenv("GITHUB_TOKEN","") RAG_URL = os.getenv("RAG_URL", "http://rag-server:8001") +KIWIX_URL = os.getenv("KIWIX_URL", "http://kiwix:80") mcp = FastMCP("local-dev-tools") @@ -180,6 +181,27 @@ def github_api(method: str, endpoint: str, body: str = "") -> str: except Exception: return r.text +# ── Kiwix search ────────────────────────────────────────────────────────────── +import re as _re + +@mcp.tool() +def kiwix_search(query: str, books: str = "") -> str: + """Search offline ZIM content via Kiwix (Wikipedia, Stack Overflow, etc). + books: optional comma-separated book names to filter, e.g. 'stackoverflow.com_en_all'. + Leave empty to search all ZIMs.""" + params = {"pattern": query, "lang": ""} + if books: + params["books"] = books + try: + r = httpx.get(f"{KIWIX_URL}/search", params=params, timeout=15) + # strip HTML tags to get plain text + text = _re.sub(r"<[^>]+>", " ", r.text) + text = _re.sub(r"&[a-zA-Z]+;", " ", text) + text = _re.sub(r"\s{2,}", "\n", text).strip() + return text[:6000] or "(no results)" + except Exception as e: + return f"[kiwix error] {e}" + # ── RAG ingest ──────────────────────────────────────────────────────────────── @mcp.tool() def ingest_repo(url: str, name: str = "", branch: str = "main") -> str: